-- Logs begin at Mon 2023-07-17 03:02:48 PKT, end at Mon 2023-07-17 11:23:51 PKT. --
Jul 17 03:02:48 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:48.981Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_194_3_metric_majorCompactionTime_75th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689544907438
Jul 17 03:02:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:48.997Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_storeCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"default\", project=\"OLAP\", project_id=\"3\", region=\"52dad430b77559a5c67aff9abd0233a8\", table=\"index_time_default\", type=\"index\"}" timestamp=1689544907438
Jul 17 03:02:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:49.046Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_numReferenceFiles\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"default\", project=\"OLAP\", project_id=\"3\", region=\"d83cc1aea06dd989e81e7b9e4481ca14\", table=\"index_filename_default\", type=\"index\"}" timestamp=1689544907438
Jul 17 03:02:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:49.093Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_putCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"21428ed55ae706e4f8cb503cd0824bec\", table=\"firewall_hos_bucket\", type=\"data\"}" timestamp=1689544907438
Jul 17 03:02:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:49.148Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_BlockedFlushes_max\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Memory\"}" timestamp=1689544907438
Jul 17 03:02:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:49.174Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_session_record_hos_bucket_metric_tableWriteQueryPerSecond_15min_rate\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689544907438
Jul 17 03:02:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:49.228Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_2_metric_flushMemstoreSize_mean\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689544907438
Jul 17 03:02:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:49.248Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_2_metric_compactionOutputFileCount_75th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689544907438
Jul 17 03:02:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:49.264Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_filteredReadRequestCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"session_record_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"335426bb6cce7e0c02759c3d6e2ad3cb\", table=\"index_filename_session_record_hos_bucket\", type=\"index\"}" timestamp=1689544907438
Jul 17 03:02:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:49.295Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_3_metric_deleteTime_25th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689544907438
Jul 17 03:02:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:49.458Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_hbase_table_rsgroup_metric_compactionTime_median\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689544907438
Jul 17 03:02:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:49.490Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_session_record_hos_bucket_metric_flushMemstoreSize_median\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689544907438
Jul 17 03:02:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:49.528Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_storeFileSize\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", namespace=\"SYSTEM\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"5a8f51f1ceb29e2c5e26a5988d4dce97\", table=\"LOG\", type=\"data\"}" timestamp=1689544907438
Jul 17 03:02:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:49.544Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_193_1_metric_majorCompactionOutputSize_99th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689544907438
Jul 17 03:02:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:49.550Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_appendCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"default\", project=\"OLAP\", project_id=\"3\", region=\"52dad430b77559a5c67aff9abd0233a8\", table=\"index_time_default\", type=\"index\"}" timestamp=1689544907438
Jul 17 03:02:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:49.555Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"vmtoolsd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689544901057
Jul 17 03:02:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:49.561Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx10\", asset_id=\"109\", cpu=\"71\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544901797
Jul 17 03:02:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:49.565Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_session_record_hos_bucket_metric_majorCompactionTime_95th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689544907438
Jul 17 03:02:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:49.574Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx10\", asset_id=\"109\", cpu=\"121\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544901797
Jul 17 03:02:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:49.575Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_lookups_total\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"ipmi-exporter\"}" timestamp=1689544904468
Jul 17 03:02:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:49.583Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_1_metric_compactionOutputSize_75th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689544907438
Jul 17 03:02:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:49.593Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_owner\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", owner_is_controller=\"true\", owner_kind=\"Job\", owner_name=\"helm-install-systemd-exporter\", pod=\"helm-install-systemd-exporter-vq7tn\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"8760a158-2fa4-4332-99b5-4233fdf937ac\"}" timestamp=1689544899668
Jul 17 03:02:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:49.595Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_compactionsFailedCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"assessment_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"73b5ff36af1eb934d5c67441ee3e9bca\", table=\"index_time_assessment_hos_bucket\", type=\"index\"}" timestamp=1689544907438
Jul 17 03:02:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:49.613Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Sink_ageOfLastAppliedOp_99th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Replication\"}" timestamp=1689544907438
Jul 17 03:02:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:49.630Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_session_record_hos_bucket_metric_deleteBatchTime_99_9th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689544907438
Jul 17 03:02:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:49.762Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_session_record_hos_bucket_metric_compactionInputFileCount_num_ops\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689544907438
Jul 17 03:02:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:49.792Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", pod=\"tsg-traffic-engine-numa-0-firewall-7886f9896b-4qb7z\", project=\"TSG-OS\", project_id=\"7\", reason=\"NodeAffinity\", service=\"prometheus-kube-state-metrics\", uid=\"8bb79bea-895f-4fbf-bba7-ff346d91b1a0\"}" timestamp=1689544899668
Jul 17 03:02:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:49.806Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2did-scsi\\\\x2d362cea7f0add58e002c2705eddd9fc7f5\\\\x2dpart5.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689544901057
Jul 17 03:02:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:49.820Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_incrementCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"proxy_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"7c11fc38fd8ff13a4c7bd76eeafe3b5e\", table=\"index_filename_proxy_hos_bucket\", type=\"index\"}" timestamp=1689544907438
Jul 17 03:02:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:49.921Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethd981cb42\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544901797
Jul 17 03:02:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:49.943Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_carrier_total\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth9e910281\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544901797
Jul 17 03:02:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:49.951Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_errors_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", interface=\"ens6f0np0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689544906183
Jul 17 03:02:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:49.968Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_annotations\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_endpoint=\"systemd-exporter\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689544901568
Jul 17 03:02:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:49.982Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_persistentvolume_labels\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", persistentvolume=\"pvc-52eb2107-7381-4fbb-a309-9ebeba926842\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689544901568
Jul 17 03:02:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:50.019Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-8bb79bea\\\\x2d895f\\\\x2d4fbf\\\\x2dbba7\\\\x2dff346d91b1a0-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2dhmk76.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689544901057
Jul 17 03:02:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:50.026Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"kube-state-metrics\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Sync\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689544892347
Jul 17 03:02:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:50.035Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-netns-cni\\\\x2df8939315\\\\x2d2758\\\\x2de501\\\\x2d8292\\\\x2d9a2eb49e1d17.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689544901057
Jul 17 03:02:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:50.036Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_4_metric_compactionTime_mean\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689544907438
Jul 17 03:02:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:50.079Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_io_current\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/1d0a9283baaeb68215db2c28cfe83a4ec2014931320cc1ec38d111b6cd89d7b3/shm\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689544902786
Jul 17 03:02:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:50.217Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544906709
Jul 17 03:02:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:50.270Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_virtual_memory_max_bytes\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sshd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689544906554
Jul 17 03:02:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:50.329Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Pkt_Inbound\", app_name=\"sapp4\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", vsysId=\"1\"}" timestamp=1689544908675
Jul 17 03:02:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:50.458Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_io_current\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"overlay_0-161\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689544902786
Jul 17 03:02:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:50.475Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_usage_bytes\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/3ef3636b7f5d7d7024841cdb198e770c0990da29955e2d43716f9fd969032bfa/shm\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689544902786
Jul 17 03:02:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:50.484Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_oom_events_total\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"prometheus-pushgateway\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689544890457
Jul 17 03:02:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:50.503Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_writes_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"overlay_0-334\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689544902786
Jul 17 03:02:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:50.536Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_info\", asset=\"twa-tsgx08\", asset_id=\"105\", configmap=\"chart-content-prometheus\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689544901568
Jul 17 03:02:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:50.553Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"machine_memory_bytes\", asset=\"twa-tsgx10\", asset_id=\"109\", boot_id=\"fa7a7e46-566a-4482-ac35-d9546cffd1ae\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", machine_id=\"254e080b2ae844f980b5182b92cc360f\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", system_uuid=\"4c4c4544-0046-5410-8031-cac04f324733\"}" timestamp=1689544899050
Jul 17 03:02:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:50.597Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"tmpfs\", name=\"var-lib-kubelet-pods-ee4ec55a\\\\x2df496\\\\x2d4629\\\\x2db299\\\\x2dcf918d7658cb-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2d7ccbd.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689544906554
Jul 17 03:02:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:50.631Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_frequency_min_hertz\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"40\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544906709
Jul 17 03:02:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:50.639Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_errors_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", interface=\"cnibr0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689544906183
Jul 17 03:02:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:50.658Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"13\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544906709
Jul 17 03:02:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:50.681Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_status_completion_time\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", job_name=\"helm-install-prometheus\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689544901568
Jul 17 03:02:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:50.775Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"twa-tsgx10\", asset_id=\"109\", cpu=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544901797
Jul 17 03:02:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:50.788Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_read_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"overlay_0-141\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689544902786
Jul 17 03:02:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:50.820Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_fifo_total\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"cnibr0\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544901797
Jul 17 03:02:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:50.849Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-a2024985\\\\x2dd999\\\\x2d4ae2\\\\x2d847f\\\\x2da93fc2842f87-volumes-kubernetes.io\\\\x7esecret-cm\\\\x2dcache\\\\x2dextra\\\\x2dsecrets.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689544901057
Jul 17 03:02:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:50.867Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_errs_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"eno36np3\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544906709
Jul 17 03:02:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:51.026Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-virtual-net-virtio_dign_s.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689544901057
Jul 17 03:02:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:51.071Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"lb-tcp-80\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"iowaiting\"}" timestamp=1689544885139
Jul 17 03:02:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:51.084Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_data_received_bytes_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_8\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544906709
Jul 17 03:02:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:51.087Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"2\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544906709
Jul 17 03:02:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:51.109Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_frequency_min_hertz\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"68\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544906709
Jul 17 03:02:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:51.128Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"4\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544906709
Jul 17 03:02:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:51.196Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db50\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689544907556
Jul 17 03:02:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:51.693Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_reservation_limit_bytes\", asset=\"twa-tsgx08\", asset_id=\"105\", container=\"local-path-provisioner\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689544907561
Jul 17 03:02:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:51.744Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"twa-tsgx08\", asset_id=\"105\", condition=\"false\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"svclb-traefik-8650ed90-pfkvd\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"b1413324-fd7a-4809-abab-764ba720b05d\"}" timestamp=1689544901568
Jul 17 03:02:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:52.240Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"tomcat_global_request_seconds_sum\", application=\"bifang_api\", asset=\"pcap-kvm-cm001\", asset_id=\"45\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"CM-API-pcap-kvm-cm001\", endpoint_id=\"285\", module=\"CM-API\", module_id=\"55\", name=\"http-nio-8080\", nz_agent_id=\"3\", parent_asset=\"pcap-dellr740-pve001\", parent_asset_id=\"43\", project=\"Central-Management\", project_id=\"5\"}" timestamp=1689544910877
Jul 17 03:02:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:52.619Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db77\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689544899367
Jul 17 03:02:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:52.655Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", cpu=\"87\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544909337
Jul 17 03:02:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:52.662Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_limit_bytes\", asset=\"pcap-tsgx11\", asset_id=\"115\", container=\"kube-multus\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689544910039
Jul 17 03:02:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:52.669Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_avail_bytes\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda5\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", fstype=\"ext4\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/var\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544909337
Jul 17 03:02:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:52.706Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", cpu=\"109\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544909337
Jul 17 03:02:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:52.952Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"pcap-dellr740-dt002\", asset_id=\"33\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"NC-Gohangout-pcap-dellr740-dt002\", endpoint_id=\"173\", module=\"DC-Gohangout\", module_id=\"35\", name=\"20\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", type=\"Processor\"}" timestamp=1689544912666
Jul 17 03:02:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:53.101Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_owner\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", job_name=\"helm-install-configmap-reloader\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", owner_is_controller=\"<none>\", owner_kind=\"<none>\", owner_name=\"<none>\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689544908950
Jul 17 03:02:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:53.214Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", cpu=\"26\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544909337
Jul 17 03:02:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:53.233Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", cpu=\"64\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544909337
Jul 17 03:02:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:53.298Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_carrier_total\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth4e8b65d2\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544903391
Jul 17 03:02:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:53.310Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"vetheb22b999\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544908328
Jul 17 03:02:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:53.312Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", cpu=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544909337
Jul 17 03:02:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:53.317Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Tcp_Pkt\", app_name=\"sapp4\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", vsysId=\"1\"}" timestamp=1689544900470
Jul 17 03:02:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:53.336Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_reads_bytes_total\", asset=\"msh-tsgx12\", asset_id=\"113\", container=\"metrics-server\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689544891374
Jul 17 03:02:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:53.343Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_packets_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth4cc39890\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544909337
Jul 17 03:02:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:53.345Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"getty.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"target\"}" timestamp=1689544911179
Jul 17 03:02:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:53.355Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_owner\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", owner_is_controller=\"true\", owner_kind=\"ReplicaSet\", owner_name=\"ipmi-exporter-6f8464dd94\", pod=\"ipmi-exporter-6f8464dd94-792kb\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"da81ea3f-d797-4414-9e38-bc69ec182d3b\"}" timestamp=1689544908663
Jul 17 03:02:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:53.383Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", cpu=\"119\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544909337
Jul 17 03:02:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:53.390Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-30711841efb36ecd67334a65adabc8557e2e12a0d6c7d7f17ad45dcb53a83605-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689544905496
Jul 17 03:02:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:53.431Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"msh-tsgx08\", asset_id=\"65\", condition=\"unknown\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"svclb-traefik-ab31cfb4-hhks6\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"44ba41ae-6294-4471-a8bc-5c98a2a4eb5d\"}" timestamp=1689544908950
Jul 17 03:02:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:53.437Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_name_assign_type\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth401fe2c4\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544903391
Jul 17 03:02:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:53.487Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx08\", asset_id=\"65\", container=\"coredns\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Sync\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689544884482
Jul 17 03:02:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:53.500Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", cpu=\"100\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544909337
Jul 17 03:02:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:53.521Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_changes_total\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"lo\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544908328
Jul 17 03:02:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:53.537Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db67\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689544899367
Jul 17 03:02:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:53.562Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"multipathd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689544912135
Jul 17 03:02:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:53.563Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_alloc_bytes\", asset=\"pcap-dellr740-dt002\", asset_id=\"33\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"NC-Gohangout-pcap-dellr740-dt002\", endpoint_id=\"173\", module=\"DC-Gohangout\", module_id=\"35\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689544912666
Jul 17 03:02:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:53.589Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_bytes_total\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"cadvisor\", interface=\"eth0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689544885787
Jul 17 03:02:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:53.647Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_completion_time\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-configmap-reloader-8w6df\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"82efaba6-ab4c-424c-a668-60c462c0aed4\"}" timestamp=1689544908950
Jul 17 03:02:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:53.672Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-platform-serial8250-tty-ttyS2.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689544905496
Jul 17 03:02:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:53.696Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_scrape_collector_duration_seconds\", asset=\"msh-tsgx08\", asset_id=\"65\", collector=\"sockstat\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544908328
Jul 17 03:02:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:53.706Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_running\", asset=\"msh-tsgx08\", asset_id=\"65\", container=\"telegraf-security\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", pod=\"tsg-traffic-engine-numa-1-firewall-686c568b7d-k5nxx\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"a7dd7c83-1f08-4d39-a9a0-eff0bdbd01be\"}" timestamp=1689544908950
Jul 17 03:02:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:53.767Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_virtual_memory_bytes\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"crond.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689544912135
Jul 17 03:02:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:53.771Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kdump.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"service\"}" timestamp=1689544912135
Jul 17 03:02:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:53.844Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"k3s.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"service\"}" timestamp=1689544912135
Jul 17 03:02:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:53.901Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_info\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_endpoint=\"shaping-announce-port-tsg-traffic-engine-numa-0\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689544904478
Jul 17 03:02:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:53.935Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", phase=\"Failed\", pod=\"ipmi-exporter-6f8464dd94-ws8dm\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"e499bad4-c3e0-46d2-8043-b45c617d5590\"}" timestamp=1689544904478
Jul 17 03:02:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:53.965Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_packets_dropped_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"cadvisor\", interface=\"br_dign_s\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689544904172
Jul 17 03:02:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:53.990Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_ports\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_endpoint=\"prometheus-pushgateway\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", port_name=\"http\", port_number=\"9091\", port_protocol=\"TCP\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689544902058
Jul 17 03:02:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:54.096Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_heap_inuse_bytes\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689544898659
Jul 17 03:02:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:54.154Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_swap_limit_bytes\", asset=\"pcap-tsgx07\", asset_id=\"85\", container=\"telegraf-security\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689544910608
Jul 17 03:02:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:54.154Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx12\", asset_id=\"113\", container=\"ipmi-exporter\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Discard\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689544884938
Jul 17 03:02:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:54.184Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", phase=\"Failed\", pod=\"kubernetes-dashboard-66dd5b456-rmvl8\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"d360d8a3-9aa3-40d2-ad94-ccc470a435ce\"}" timestamp=1689544902058
Jul 17 03:02:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:54.201Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_hwmon_sensor_label\", asset=\"pcap-tsgx07\", asset_id=\"85\", chip=\"pci0000:00_0000:00:19_3\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", label=\"tccd1\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", sensor=\"temp3\", service=\"prometheus-node-exporter\"}" timestamp=1689544899409
Jul 17 03:02:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:54.224Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db157\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689544905184
Jul 17 03:02:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:54.295Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cpu=\"0\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544899409
Jul 17 03:02:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:54.551Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_net_dev_group\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"vethebc85272\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544908328
Jul 17 03:02:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:54.588Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_fifo_total\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"eno1\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544908328
Jul 17 03:02:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:54.620Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_threads\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689544896138
Jul 17 03:02:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:54.659Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_owner\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", owner_is_controller=\"true\", owner_kind=\"ReplicaSet\", owner_name=\"prometheus-kube-state-metrics-6b768c7bf8\", pod=\"prometheus-kube-state-metrics-6b768c7bf8-bvmvm\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"3f9ac58b-863e-48ca-98d2-735c5ebff595\"}" timestamp=1689544908950
Jul 17 03:02:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:54.727Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_netstat_UdpLite_InErrors\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544908328
Jul 17 03:02:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:54.751Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_frame_total\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"eno1\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544908328
Jul 17 03:02:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:54.755Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-ens6f0np0.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689544912135
Jul 17 03:02:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:54.802Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db64\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689544910108
Jul 17 03:02:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:54.849Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Udp_Offload_STM\", app_name=\"sapp4\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", vsysId=\"1\"}" timestamp=1689544901765
Jul 17 03:02:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:54.870Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-897e78337f0b38a3f14c6bb329ef34eba269643e95d2513a0b17caa9d0182a4b.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"scope\"}" timestamp=1689544900884
Jul 17 03:02:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:54.885Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_file_descriptors\", asset=\"msh-tsgx08\", asset_id=\"65\", container=\"dign-server-web\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689544896020
Jul 17 03:02:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:54.904Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"msh-tsgx08\", asset_id=\"65\", container=\"traefik\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"cadvisor\", failure_type=\"pgmajfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"container\"}" timestamp=1689544896421
Jul 17 03:02:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:54.946Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_ulimits_soft\", asset=\"msh-tsgx08\", asset_id=\"65\", container=\"kube-multus\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", ulimit=\"max_open_files\"}" timestamp=1689544895274
Jul 17 03:02:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:55.157Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"msh-tsgx12\", asset_id=\"113\", cpu=\"58\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544903391
Jul 17 03:02:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:55.344Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_resident_memory_bytes\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"irqbalance.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689544900884
Jul 17 03:02:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:55.362Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-burstable.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"slice\"}" timestamp=1689544900884
Jul 17 03:02:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:55.384Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_node_status_condition\", asset=\"pcap-tsgx07\", asset_id=\"85\", condition=\"Ready\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", node=\"localhost\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"false\"}" timestamp=1689544902058
Jul 17 03:02:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:55.397Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cpu=\"44\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544899409
Jul 17 03:02:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:55.504Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_state_started\", asset=\"pcap-tsgx07\", asset_id=\"85\", container=\"traefik\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"traefik-64f55bb67d-bcw8n\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"b9686156-a602-47ce-b798-d231f10c8f92\"}" timestamp=1689544902058
Jul 17 03:02:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:56.346Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-900223f6\\\\x2d7495\\\\x2d4cd9\\\\x2db023\\\\x2d184522fd9a95-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2dwmk4d.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689544903372
Jul 17 03:02:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:56.634Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"25\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544905883
Jul 17 03:02:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:56.645Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_write_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda4\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689544910655
Jul 17 03:02:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:56.654Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_swap\", asset=\"pcap-tsgx03\", asset_id=\"73\", container=\"prometheus-server-configmap-reload\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689544895072
Jul 17 03:02:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:56.750Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"74\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544905883
Jul 17 03:02:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:57.500Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"114\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544905883
Jul 17 03:02:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:57.585Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db111\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689544906475
Jul 17 03:02:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:57.611Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"initrd-parse-etc.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689544903372
Jul 17 03:02:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:57.622Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-2c2b288b52bbbd64ff60706f898723773a0ea70b14ca0ed436f26937245a8811.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"scope\"}" timestamp=1689544903372
Jul 17 03:02:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:57.796Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_files\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda3\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", fstype=\"ext4\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/update\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544905883
Jul 17 03:02:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:57.855Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"ext4\", name=\"var-lib-kubelet-pods-aebe8d09\\\\x2d6d5d\\\\x2d4eda\\\\x2db90d\\\\x2d7be24e2da709-volume\\\\x2dsubpaths-config\\\\x2dvolume-firewall-1.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689544903372
Jul 17 03:02:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:57.871Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_free_bytes\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sdb1\", endpoint=\"OLAP-Node-Exporter-pcap-dellr740-dt001\", endpoint_id=\"251\", fstype=\"xfs\", module=\"OLAP-Node-Exporter\", module_id=\"51\", mountpoint=\"/home\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689544917557
Jul 17 03:02:58 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:58.990Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", cpu=\"33\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"OLAP-Node-Exporter-pcap-dellr740-dt001\", endpoint_id=\"251\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689544917557
Jul 17 03:02:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:59.754Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_carrier_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"br_dign_s\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544909083
Jul 17 03:02:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:59.756Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_annotations\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.mrzcpd-exporter.v1\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689544907425
Jul 17 03:02:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:59.760Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_fifo_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"vethd3934cae\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544909083
Jul 17 03:02:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:59.774Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"43\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544909083
Jul 17 03:02:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:59.778Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"118\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544909083
Jul 17 03:02:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:59.799Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db98\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689544905071
Jul 17 03:02:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:59.818Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-infiniband-umad7.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689544904879
Jul 17 03:02:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:59.835Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"watchdog.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689544904879
Jul 17 03:02:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:02:59.844Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_queue_length\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth9b710207\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544909083
Jul 17 03:03:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:00.062Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_avail_bytes\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"shm\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/af2d86ef683340f2922e5c0283f1bb12dd7878f090cc7dc66cbd14984f57924e/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544909083
Jul 17 03:03:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:00.080Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link_mode\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth8bc626f9\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544909083
Jul 17 03:03:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:00.097Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_fifo_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens7f0np0\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544909083
Jul 17 03:03:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:00.103Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_created\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", job_name=\"helm-install-traefik-crd\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689544907425
Jul 17 03:03:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:00.135Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"NetworkManager.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"service\"}" timestamp=1689544904879
Jul 17 03:03:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:00.261Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"68\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544909083
Jul 17 03:03:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:00.338Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"119\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544909083
Jul 17 03:03:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:00.373Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db87\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689544905071
Jul 17 03:03:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:00.398Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cpupower.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"service\"}" timestamp=1689544904879
Jul 17 03:03:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:00.925Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-14c3fe4fcd9dc07ae7ffafd062c5ba6bec755d239f97cf254e18db2c53443639.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"scope\"}" timestamp=1689544904879
Jul 17 03:03:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:01.243Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_owner\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", owner_is_controller=\"true\", owner_kind=\"ReplicaSet\", owner_name=\"tsg-traffic-engine-numa-0-firewall-57597b866c\", pod=\"tsg-traffic-engine-numa-0-firewall-57597b866c-6hbxk\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"55b23504-098e-4605-9a10-91ab01c5f1f9\"}" timestamp=1689544907425
Jul 17 03:03:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:01.409Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_annotations\", asset=\"msh-tsgx01\", asset_id=\"51\", configmap=\"prometheus-alertmanager\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689544907425
Jul 17 03:03:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:01.443Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", phase=\"Pending\", pod=\"systemd-exporter-7f549d9485-qmd8f\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"45b995ae-405c-4e5d-b21a-3830d4e78e4d\"}" timestamp=1689544907425
Jul 17 03:03:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:01.466Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"25\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544909083
Jul 17 03:03:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:01.627Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_address_not_ready\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_endpoint=\"prometheus-server\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689544907425
Jul 17 03:03:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:02.447Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_fifo_total\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth74f1e741\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544918318
Jul 17 03:03:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:02.525Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_packets_dropped_total\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", interface=\"eth0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689544901047
Jul 17 03:03:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:02.606Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2did-scsi\\\\x2d362cea7f084826d002c2b6d2f9e5d290c\\\\x2dpart4.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689544908447
Jul 17 03:03:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:02.640Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"mnt-onie\\\\x2dboot.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689544908447
Jul 17 03:03:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:02.658Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-vethad44a482.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689544908447
Jul 17 03:03:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:02.715Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:80-0000:80:01.1-0000:81:00.0-host0-target0:2:0-0:2:0:0-block-sda-sda5.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689544908447
Jul 17 03:03:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:02.729Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_scrape_collector_duration_seconds\", asset=\"msh-tsgx02\", asset_id=\"63\", collector=\"textfile\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544918318
Jul 17 03:03:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:02.797Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_created\", asset=\"msh-tsgx02\", asset_id=\"63\", configmap=\"kube-root-ca.crt\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689544911412
Jul 17 03:03:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:02.799Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_created\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"kube-dns\", uid=\"2209f98a-1b5b-4f6d-a59f-cc1dda458252\"}" timestamp=1689544911412
Jul 17 03:03:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:02.814Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_daemonset_created\", asset=\"msh-tsgx02\", asset_id=\"63\", daemonset=\"prometheus-node-exporter\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689544911412
Jul 17 03:03:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:02.911Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"eno1\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544918318
Jul 17 03:03:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:02.966Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_max_fds\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"irqbalance.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689544908447
Jul 17 03:03:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:03.042Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-bf01a0b5af7e76dd03aadedc4cc2c5899b1f69e8a2f6c08decf3631271891f8f-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689544908447
Jul 17 03:03:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:03.118Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"8\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689544918318
Jul 17 03:03:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:03.421Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"NetworkManager-wait-online.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"service\"}" timestamp=1689544908447
Jul 17 03:03:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:03.645Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_memory_AnonPages_bytes\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544918318
Jul 17 03:03:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:03.948Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_load_average_10s\", asset=\"msh-tsgx02\", asset_id=\"63\", container=\"metrics\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689544911038
Jul 17 03:03:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:04.155Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_spec_paused\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"configmap-reloader\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689544911412
Jul 17 03:03:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:04.540Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", pod=\"dashboard-metrics-scraper-5b94897dc9-gkbdq\", project=\"TSG-OS\", project_id=\"7\", reason=\"NodeLost\", service=\"prometheus-kube-state-metrics\", uid=\"03938281-e348-4392-a795-f2d53f05d546\"}" timestamp=1689544911412
Jul 17 03:03:05 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:05.026Z caller=dedupe.go:112 component=remote level=info remote_name=52db6f url=http://10.10.10.159:10090/nz-agent/prometheus/proxy/api/v1/write msg="Dropped sample for series that was not explicitly dropped via relabelling" ref=4436650
Jul 17 03:03:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:06.613Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cpu=\"115\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544923910
Jul 17 03:03:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:06.633Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"mrzcpd_hugepages_setup.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"service\"}" timestamp=1689544913582
Jul 17 03:03:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:06.649Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sshd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"service\"}" timestamp=1689544913582
Jul 17 03:03:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:06.661Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"29\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689544923910
Jul 17 03:03:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:06.682Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cpu=\"20\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544923910
Jul 17 03:03:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:06.702Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:c0-0000:c0:01.1-0000:c1:00.1-infiniband_mad-umad9.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689544913582
Jul 17 03:03:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:06.951Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_max_fds\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"chronyd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689544913582
Jul 17 03:03:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:07.005Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"tsg_master_updating\", app_name=\"tsg_master\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"traffic-engine\", line_name=\"fail\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", table_name=\"ctrl_stat\", vsysId=\"1\"}" timestamp=1689544924123
Jul 17 03:03:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:07.466Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db148\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689544918290
Jul 17 03:03:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:07.497Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx01\", asset_id=\"91\", container=\"redis\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Write\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689544916850
Jul 17 03:03:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:07.522Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_address_available\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_endpoint=\"prometheus-node-exporter\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689544921227
Jul 17 03:03:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:07.552Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_labels\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"mrzcpd-exporter-5cf7f967cc\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689544921227
Jul 17 03:03:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:07.712Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dracut-initqueue.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"service\"}" timestamp=1689544913582
Jul 17 03:03:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:07.738Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"twa-tsgx01\", asset_id=\"91\", condition=\"true\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"metrics-server-54dc485875-l4kw8\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"e96787f3-8c20-4c5e-a395-ec874ca09072\"}" timestamp=1689544921227
Jul 17 03:03:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:07.767Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_created\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"chart-values-mrzcpd-exporter\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689544921227
Jul 17 03:03:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:08.077Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-110199507a0e8f844c99f44197d6d570249bc2279237b31caaf73fc07d23922d-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689544913582
Jul 17 03:03:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:08.165Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_dormant\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth26ef770a\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544923910
Jul 17 03:03:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:08.216Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_persistentvolume_labels\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", persistentvolume=\"pvc-9895065e-9211-44f6-bb57-487c596ff549\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689544921227
Jul 17 03:03:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:08.325Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-773f634ef002e77afc003e5440e1165db1ea1674b52416a01d116bc1bbbe6302.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"scope\"}" timestamp=1689544913582
Jul 17 03:03:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:08.675Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"twa-tsgx01\", asset_id=\"91\", container=\"dign-server-web\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"cadvisor\", failure_type=\"pgfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"hierarchy\"}" timestamp=1689544911934
Jul 17 03:03:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:08.697Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_packets_dropped_total\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"cadvisor\", interface=\"eth0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689544911845
Jul 17 03:03:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:08.715Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_threads_max\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689544922963
Jul 17 03:03:09 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:09.468Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx11\", asset_id=\"119\", cpu=\"18\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx11\", endpoint_id=\"461\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544918614
Jul 17 03:03:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:10.206Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_STATS_metric_majorCompactionOutputFileCount_min\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689544928807
Jul 17 03:03:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:11.336Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_numFilesCompactedCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"90fbc21026ad7a40f47226b9ac98913b\", table=\"troubleshooting_hos_bucket\", type=\"data\"}" timestamp=1689544930147
Jul 17 03:03:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:11.346Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_compactionsCompletedCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"rtp_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"12da2e465ea17433fca2e4a9ecdfa346\", table=\"index_partfile_rtp_hos_bucket\", type=\"index\"}" timestamp=1689544930147
Jul 17 03:03:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:11.364Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_2_metric_putBatchTime_max\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689544930147
Jul 17 03:03:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:11.432Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_STATS_metric_appendTime_mean\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689544930147
Jul 17 03:03:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:11.556Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_2_metric_putBatchTime_90th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689544930147
Jul 17 03:03:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:11.712Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_STATS_metric_putBatchTime_median\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689544930147
Jul 17 03:03:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:11.759Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_194_4_metric_scanTime_99th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689544930147
Jul 17 03:03:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:11.793Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_3_metric_compactionTime_98th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689544930147
Jul 17 03:03:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:12.130Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_numReferenceFiles\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", namespace=\"SYSTEM\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"ed92f2fb3f7db9aa4461f8283225dec7\", table=\"LOG\", type=\"data\"}" timestamp=1689544930147
Jul 17 03:03:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:12.324Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-dellr740-dt002\", asset_id=\"39\", cpu=\"1\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"NC-Gohangout-twa-dellr740-dt002\", endpoint_id=\"179\", mode=\"nice\", module=\"DC-Gohangout\", module_id=\"35\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689544931525
Jul 17 03:03:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:12.676Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Bulkload_count\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Server\"}" timestamp=1689544930147
Jul 17 03:03:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:12.753Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_1_metric_majorCompactionOutputFileCount_95th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689544930147
Jul 17 03:03:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:12.789Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_lastMajorCompactionAge\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"proxy_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"513c53b10d036c82752eb4ff5811ee84\", table=\"index_partfile_proxy_hos_bucket\", type=\"index\"}" timestamp=1689544930147
Jul 17 03:03:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:15.591Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_annotations\", asset=\"twa-tsgx07\", asset_id=\"103\", configmap=\"telegraf-tsg-traffic-engine-numa-1\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689544928076
Jul 17 03:03:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:15.603Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_namespace_status_phase\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Terminating\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689544928076
Jul 17 03:03:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:15.624Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"35\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544921230
Jul 17 03:03:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:15.637Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_drop_total\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth4a3c1040\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544921230
Jul 17 03:03:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:15.682Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"overlay\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-2e122063ee2c201eace04f25ac4f6d02502e3258d1a3613479961720dee5e2b1-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689544927046
Jul 17 03:03:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:15.758Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"overlay\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-c5658639abd2596fbd9b1f342d2074cb6f5a9658f302adc221d02a3812aa069f-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689544927046
Jul 17 03:03:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:15.798Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"14\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544921230
Jul 17 03:03:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:15.819Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-6adadf551ce1dda054cd6812f4f516a16930694b2bfd6009c53c19ada91f008b-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689544927046
Jul 17 03:03:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:16.004Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_namespace_status_phase\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-public\", nz_agent_id=\"3\", phase=\"Active\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689544928076
Jul 17 03:03:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:16.126Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"mlocate-updatedb.timer\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"timer\"}" timestamp=1689544927046
Jul 17 03:03:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:16.323Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"12\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544921230
Jul 17 03:03:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:16.353Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_labels\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"chart-values-tsg-traffic-engine-numa-1\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689544928076
Jul 17 03:03:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:16.370Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"86\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544921230
Jul 17 03:03:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:16.378Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_scrape_collector_duration_seconds\", asset=\"twa-tsgx07\", asset_id=\"103\", collector=\"pressure\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544921230
Jul 17 03:03:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:16.474Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_reads_total\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"overlay_0-47\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689544922355
Jul 17 03:03:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:16.497Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_writes_bytes_total\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689544927333
Jul 17 03:03:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:16.536Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_spec_replicas\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"metrics-server\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689544928076
Jul 17 03:03:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:16.546Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_spec_paused\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"dashboard-metrics-scraper\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689544928076
Jul 17 03:03:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:19.423Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_errs_total\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth7064a579\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544937612
Jul 17 03:03:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:19.437Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_fifo_total\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth7064a579\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544937612
Jul 17 03:03:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:19.448Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-ens6f0np0.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689544924356
Jul 17 03:03:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:19.561Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_timex_offset_seconds\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544937612
Jul 17 03:03:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:19.632Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_dormant\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens2f0np0\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544937612
Jul 17 03:03:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:20.484Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_up\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth96cee52c\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544928263
Jul 17 03:03:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:20.561Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2dpartuuid-ecf82303\\\\x2de12f\\\\x2d41f7\\\\x2da426\\\\x2da41b6afa09d9.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689544930657
Jul 17 03:03:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:20.585Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-ens7f1np1.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689544930657
Jul 17 03:03:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:20.656Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-33ff383b25c3d998256344f7dd955a072c9a1da15be27646815cf30a5b73efdc.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"scope\"}" timestamp=1689544930657
Jul 17 03:03:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:20.744Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_down_changes_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens7f0np0\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544928263
Jul 17 03:03:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:20.770Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db102\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689544934840
Jul 17 03:03:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:20.826Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_node_status_capacity\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", node=\"localhost\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", resource=\"cpu\", service=\"prometheus-kube-state-metrics\", unit=\"core\"}" timestamp=1689544933949
Jul 17 03:03:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:21.127Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled_time\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"local-path-provisioner-957fdf8bc-6hrfp\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"03ed11d6-2d8c-4174-885e-b558bd212ab7\"}" timestamp=1689544933828
Jul 17 03:03:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:21.128Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_speed_bytes\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"eno33np0\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544928263
Jul 17 03:03:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:21.145Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_owner\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", owner_is_controller=\"true\", owner_kind=\"Deployment\", owner_name=\"prometheus-kube-state-metrics\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"prometheus-kube-state-metrics-6b768c7bf8\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689544933828
Jul 17 03:03:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:21.185Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-virtual-net-veth96cee52c.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689544930657
Jul 17 03:03:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:21.543Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"26\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544928263
Jul 17 03:03:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:21.931Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"iscsid.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"service\"}" timestamp=1689544930657
Jul 17 03:03:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:21.944Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-0714120bee5c10508c219979d0b89963503d3f40b80e256ddae84600df3b33b1-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689544930657
Jul 17 03:03:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:22.002Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_reservation_limit_bytes\", asset=\"pcap-tsgx10\", asset_id=\"89\", container=\"kube-state-metrics\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689544935735
Jul 17 03:03:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:22.039Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544928263
Jul 17 03:03:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:22.049Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"snmptrapd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"notify\", type=\"service\"}" timestamp=1689544930657
Jul 17 03:03:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:22.118Z caller=dedupe.go:112 component=remote level=error remote_name=52db6f url=http://10.10.10.159:10090/nz-agent/prometheus/proxy/api/v1/write msg="non-recoverable error" count=1000 exemplarCount=0 err="server returned HTTP status 400 : out of order sample"
Jul 17 03:03:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:22.130Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"pcap-tsgx10\", asset_id=\"89\", condition=\"unknown\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-configmap-reloader-6hq4f\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"08021e64-c970-4f83-8ae6-ffae5f3c407b\"}" timestamp=1689544933828
Jul 17 03:03:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:22.169Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"phy_rx_bits_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", dev=\"virtio_dign_c\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"mrzcpd-exporter\"}" timestamp=1689544934036
Jul 17 03:03:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:22.331Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_mspan_sys_bytes\", asset=\"msh-dellr740-olap004\", asset_id=\"17\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Spark-msh-dellr740-olap004\", endpoint_id=\"15\", module=\"NC-Spark\", module_id=\"47\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689544942258
Jul 17 03:03:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:24.218Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_working_set_bytes\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"lb-tcp-80\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689544935243
Jul 17 03:03:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:24.220Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"61\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544940160
Jul 17 03:03:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:24.230Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"8\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544940160
Jul 17 03:03:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:24.244Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"105\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544940160
Jul 17 03:03:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:24.257Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"33\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544940160
Jul 17 03:03:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:24.288Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db107\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689544937019
Jul 17 03:03:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:24.298Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_size_bytes\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"shm\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/75f572b390f66779a4d22b438e080cf65508a2f5883664b28a67e71b1c0bc8f7/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544940160
Jul 17 03:03:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:24.319Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db213\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689544941795
Jul 17 03:03:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:24.341Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_start_time_seconds\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689544941795
Jul 17 03:03:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:24.388Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_resource_limits\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"coredns\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", node=\"localhost\", nz_agent_id=\"3\", pod=\"coredns-ff8b8fd9d-c4wjl\", project=\"TSG-OS\", project_id=\"7\", resource=\"memory\", service=\"prometheus-kube-state-metrics\", uid=\"a6dafded-0d01-479f-906c-089305c5928a\", unit=\"byte\"}" timestamp=1689544931122
Jul 17 03:03:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:24.407Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_reads_bytes_total\", asset=\"msh-tsgx09\", asset_id=\"55\", container=\"prometheus-server\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689544924795
Jul 17 03:03:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:24.427Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_resource_requests\", asset=\"msh-tsgx09\", asset_id=\"55\", container=\"coredns\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", node=\"localhost\", nz_agent_id=\"3\", pod=\"coredns-7cbc7ddf87-8lntt\", project=\"TSG-OS\", project_id=\"7\", resource=\"memory\", service=\"prometheus-kube-state-metrics\", uid=\"bfb92468-68e3-414d-a511-fe5cfafd4a5a\", unit=\"byte\"}" timestamp=1689544940068
Jul 17 03:03:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:24.428Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"coredns\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Async\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689544942403
Jul 17 03:03:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:24.447Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx09\", asset_id=\"55\", cpu=\"19\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544934790
Jul 17 03:03:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:24.468Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_readonly\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda5\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", fstype=\"ext4\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/var\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544934790
Jul 17 03:03:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:24.516Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_limit_bytes\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"kubernetes-dashboard\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689544943419
Jul 17 03:03:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:24.518Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_info\", asset=\"msh-tsgx09\", asset_id=\"55\", board_id=\"MT_0000000359\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", firmware_version=\"22.28.4000\", hca_type=\"MT4125\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544934790
Jul 17 03:03:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:24.535Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"pcap-tsgx12\", asset_id=\"117\", condition=\"Progressing\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"prometheus-alertmanager\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"unknown\"}" timestamp=1689544931122
Jul 17 03:03:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:24.541Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"msh-tsgx09\", asset_id=\"55\", cpu=\"14\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544934790
Jul 17 03:03:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:24.552Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_fifo_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"vethc96a0a69\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544940160
Jul 17 03:03:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:24.587Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db129\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689544941795
Jul 17 03:03:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:24.657Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"var-lib-kubelet-pods-f2551d6b\\\\x2d0939\\\\x2d434d\\\\x2dba32\\\\x2d2e369c56471c-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2ddmx7x.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689544938416
Jul 17 03:03:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:24.698Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db240\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689544941795
Jul 17 03:03:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:24.723Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"ext4\", name=\"var-lib-kubelet-pods-692f1a13\\\\x2d48e0\\\\x2d47b7\\\\x2dad46\\\\x2da5b348bd402b-volume\\\\x2dsubpaths-config\\\\x2dvolume-firewall-1.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689544931102
Jul 17 03:03:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:24.766Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx09\", asset_id=\"55\", cpu=\"46\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544934790
Jul 17 03:03:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:24.785Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"mnt-onie\\\\x2dboot.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689544938416
Jul 17 03:03:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:24.827Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"configmap-reloader-6c55bbc49f-fh258\", project=\"TSG-OS\", project_id=\"7\", reason=\"UnexpectedAdmissionError\", service=\"prometheus-kube-state-metrics\", uid=\"f7af3cd7-b972-4aba-a12a-7b5c827e57eb\"}" timestamp=1689544931122
Jul 17 03:03:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:24.845Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_created\", asset=\"msh-tsgx09\", asset_id=\"55\", configmap=\"multus-cni-config\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689544940068
Jul 17 03:03:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:24.951Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db116\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689544937019
Jul 17 03:03:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:24.998Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_status_active\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", job_name=\"helm-install-ipmi-exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689544931122
Jul 17 03:03:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:24.998Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-netns-cni\\\\x2da7388962\\\\x2d2db0\\\\x2d3d46\\\\x2dd37f\\\\x2dcd8de5a74997.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689544931102
Jul 17 03:03:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:25.063Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"local-path-provisioner\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Total\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689544933400
Jul 17 03:03:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:25.070Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"109\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544940160
Jul 17 03:03:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:25.117Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db207\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689544941795
Jul 17 03:03:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:25.132Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"tsg-os-HAL.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689544931102
Jul 17 03:03:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:25.657Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db13\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689544941795
Jul 17 03:03:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:25.754Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"13\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544940160
Jul 17 03:03:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:25.761Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"pcap-tsgx12\", asset_id=\"117\", condition=\"Progressing\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"configmap-reloader\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"unknown\"}" timestamp=1689544931122
Jul 17 03:03:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:25.778Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled_time\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-node-exporter-qmjq7\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"e8a13065-a344-44f4-a0e1-7369c47522e2\"}" timestamp=1689544931122
Jul 17 03:03:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:26.076Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"125\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544935810
Jul 17 03:03:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:26.077Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"102\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544935810
Jul 17 03:03:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:26.096Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"43\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544935810
Jul 17 03:03:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:26.178Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"var-lib-kubelet-pods-e2fa51ee\\\\x2d442f\\\\x2d45b7\\\\x2d9aac\\\\x2d868fa0aef812-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2ddffmb.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689544933907
Jul 17 03:03:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:26.207Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-pushgateway-bd476bcf9-zqlv8\", project=\"TSG-OS\", project_id=\"7\", reason=\"NodeAffinity\", service=\"prometheus-kube-state-metrics\", uid=\"880aa12b-51b0-4413-88cb-d06b2fc6d0c3\"}" timestamp=1689544938744
Jul 17 03:03:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:26.238Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-ecade2256a71baee22a29c7948d507742382f9d4293dc93d68f0b911800feabc-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689544933907
Jul 17 03:03:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:26.281Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_info\", address=\"bc:97:e1:c6:89:e2\", asset=\"twa-tsgx09\", asset_id=\"107\", broadcast=\"ff:ff:ff:ff:ff:ff\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"eno35np2\", duplex=\"unknown\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operstate=\"down\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544935810
Jul 17 03:03:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:26.355Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_dormant\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth3e8ea297\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544935810
Jul 17 03:03:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:26.375Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-update-done.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"service\"}" timestamp=1689544933907
Jul 17 03:03:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:26.401Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"75\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544935810
Jul 17 03:03:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:26.626Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx09\", asset_id=\"107\", container=\"prometheus-server\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Write\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689544935699
Jul 17 03:03:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:26.655Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_packets_total\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"eno34np1\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544935810
Jul 17 03:03:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:26.675Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_bytes_total\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens7f1np1\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544935810
Jul 17 03:03:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:26.742Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_owner\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", owner_is_controller=\"true\", owner_kind=\"Job\", owner_name=\"helm-install-prometheus\", pod=\"helm-install-prometheus-4sclr\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"201e71b2-29a8-486d-833c-9f1e27354310\"}" timestamp=1689544938744
Jul 17 03:03:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:26.770Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"50\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544935810
Jul 17 03:03:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:26.973Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"phy_tx_packets_total\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", dev=\"ens1f0np0\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"mrzcpd-exporter\"}" timestamp=1689544938295
Jul 17 03:03:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:27.075Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_spec_type\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"dashboard-metrics-scraper\", type=\"ClusterIP\", uid=\"850b980d-416c-47e1-8b1f-554902fcfbc6\"}" timestamp=1689544938744
Jul 17 03:03:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:27.285Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"45\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544935810
Jul 17 03:03:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:27.481Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_buck_hash_sys_bytes\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689544939446
Jul 17 03:03:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:27.528Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_terminated\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"helm\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-traefik-rdmft\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"3daa5d94-8d7c-4d71-883e-255619668328\"}" timestamp=1689544935682
Jul 17 03:03:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:27.539Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"114\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544942882
Jul 17 03:03:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:27.753Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_id\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth2e13d535\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544942882
Jul 17 03:03:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:27.784Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"80\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689544935810
Jul 17 03:03:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:27.867Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"remote-fs.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"target\"}" timestamp=1689544933907
Jul 17 03:03:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:27.894Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_device_id\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vetha8a8b4ed\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544935810
Jul 17 03:03:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:27.930Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"fcoe.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689544933907
Jul 17 03:03:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:27.950Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_mtu_bytes\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens2f1np1\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544942882
Jul 17 03:03:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:27.962Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_sector_reads_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689544943290
Jul 17 03:03:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:27.983Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_write_seconds_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"overlay_0-47\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689544943290
Jul 17 03:03:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:28.122Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"clickhouse_os_nice_time_cpu2\", asset=\"msh-dellr740-dt003\", asset_id=\"9\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Clickhouse-msh-dellr740-dt003\", endpoint_id=\"75\", module=\"NC-Clickhouse\", module_id=\"29\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689544946493
Jul 17 03:03:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:28.233Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_MemNonHeapMaxM\", asset=\"pcap-dellr740-dt002\", asset_id=\"33\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt002-Hmaster\", endpoint_id=\"345\", module=\"DC-HBase\", module_id=\"3\", name=\"JvmMetrics\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689544947136
Jul 17 03:03:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:28.364Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_multicast_packets_transmitted_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_8\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544942882
Jul 17 03:03:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:28.506Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"msh-tsgx10\", asset_id=\"69\", condition=\"Progressing\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"prometheus-alertmanager\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"unknown\"}" timestamp=1689544935682
Jul 17 03:03:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:28.520Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_annotations\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"shaping-announce-port-tsg-traffic-engine-numa-0\", uid=\"796b9ec8-8607-4de8-b81e-4476eafa0e4c\"}" timestamp=1689544935682
Jul 17 03:03:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:28.537Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_name_assign_type\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"eno1\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544942882
Jul 17 03:03:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:28.699Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_queue_length\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth2e13d535\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544942882
Jul 17 03:03:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:28.751Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"process_resident_memory_bytes\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689544939446
Jul 17 03:03:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:31.007Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db121\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689544950141
Jul 17 03:03:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:31.035Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"12\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544940939
Jul 17 03:03:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:31.139Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689544940939
Jul 17 03:03:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:31.271Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_info\", asset=\"msh-tsgx05\", asset_id=\"67\", created_by_kind=\"ReplicaSet\", created_by_name=\"prometheus-kube-state-metrics-6b768c7bf8\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", host_ip=\"10.10.20.165\", host_network=\"false\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", node=\"localhost\", nz_agent_id=\"3\", pod=\"prometheus-kube-state-metrics-6b768c7bf8-j4svp\", pod_ip=\"10.42.0.126\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"d78263bf-476b-4921-a83b-0fe401ff42c2\"}" timestamp=1689544944068
Jul 17 03:03:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:31.278Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_restart_policy\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", pod=\"tsg-traffic-engine-numa-0-firewall-76b5c5788d-2cs6q\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", type=\"Always\", uid=\"99e7b7f0-307f-4274-a670-faf429f178f7\"}" timestamp=1689544944068
Jul 17 03:03:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:31.374Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_net_dev_group\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth8d362ef6\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544940939
Jul 17 03:03:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:31.511Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"nss-lookup.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"target\"}" timestamp=1689544946907
Jul 17 03:03:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:31.536Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:e0-0000:e0:03.1-0000:e2:00.1-infiniband_mad-umad11.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689544946907
Jul 17 03:03:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:31.588Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx03\", asset_id=\"95\", container=\"prometheus-server\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Write\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689544942465
Jul 17 03:03:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:31.596Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"gssproxy.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"service\"}" timestamp=1689544941015
Jul 17 03:03:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:31.603Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_labels\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-tsg-traffic-engine-numa-1-vnr97\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"655855cf-bcb1-4cce-bea6-d09a402d65d3\"}" timestamp=1689544950436
Jul 17 03:03:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:31.608Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_labels\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"localhost.node-password.k3s\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689544950436
Jul 17 03:03:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:31.625Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_multicast_packets_received_total\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544939093
Jul 17 03:03:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:31.642Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_hwmon_temp_celsius\", asset=\"twa-tsgx03\", asset_id=\"95\", chip=\"pci0000:00_0000:00:19_3\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", sensor=\"temp1\", service=\"prometheus-node-exporter\"}" timestamp=1689544939093
Jul 17 03:03:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:31.661Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_size_bytes\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda3\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", fstype=\"ext4\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/update\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544939093
Jul 17 03:03:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:31.672Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"108\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544939093
Jul 17 03:03:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:31.706Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"59\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544939093
Jul 17 03:03:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:31.730Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_labels\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", job_name=\"helm-install-traefik-crd\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689544939276
Jul 17 03:03:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:31.738Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_info\", asset=\"msh-tsgx11\", asset_id=\"111\", configmap=\"telegraf-tsg-traffic-engine-numa-0\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689544939276
Jul 17 03:03:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:31.745Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-rdma-devices-mlx5_4:1-umad.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689544946907
Jul 17 03:03:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:31.753Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_discards_transmitted_total\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_7\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544940791
Jul 17 03:03:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:31.770Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_name_assign_type\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth018e4616\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544940791
Jul 17 03:03:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:31.819Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_spec_replicas\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"prometheus-pushgateway\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689544944068
Jul 17 03:03:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:31.823Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_symbol_error_total\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_10\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544939093
Jul 17 03:03:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:31.835Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"15\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544940939
Jul 17 03:03:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:31.840Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"88\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544940791
Jul 17 03:03:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:31.871Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_queue_length\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"virtio_dign_c\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544939093
Jul 17 03:03:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:31.886Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-virtual-net-veth662ec9f2.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689544946907
Jul 17 03:03:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:31.918Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"ostree-remount.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689544942644
Jul 17 03:03:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:31.954Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_size_bytes\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"shm\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/48229926dd9777cc5f2899b2bdbc94f22a6ffe91ea4fcbe535a19c336b569502/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544940939
Jul 17 03:03:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:32.003Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"70\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544940791
Jul 17 03:03:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:32.034Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"105\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544940939
Jul 17 03:03:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:32.036Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-8d0b64c98fabf15d237e79bd678a3ce05be5b46cbf6a80bf62ae7512cecdde67-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689544941015
Jul 17 03:03:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:32.040Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_errors_total\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"cadvisor\", interface=\"eno34np1\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689544938482
Jul 17 03:03:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:32.044Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Unknown\", pod=\"helm-install-mrzcpd-exporter-8dssx\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"24d38c73-ce01-4d33-ae4e-dc1ddfa73fb6\"}" timestamp=1689544944068
Jul 17 03:03:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:32.091Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_owner\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", owner_is_controller=\"true\", owner_kind=\"DaemonSet\", owner_name=\"kube-multus-ds\", pod=\"kube-multus-ds-8krwn\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"da4fef9f-6e38-4557-ac6b-3ab5eb04a34f\"}" timestamp=1689544950436
Jul 17 03:03:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:32.103Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_commands_duration_seconds_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cmd=\"info\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689544950141
Jul 17 03:03:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:32.167Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_bytes_total\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethb4349618\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544939093
Jul 17 03:03:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:32.204Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx05\", asset_id=\"67\", container=\"prometheus-alertmanager\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Discard\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689544925245
Jul 17 03:03:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:32.205Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db89\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689544950141
Jul 17 03:03:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:32.291Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"0\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544940791
Jul 17 03:03:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:32.309Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"42\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544940791
Jul 17 03:03:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:32.358Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Unknown\", pod=\"helm-install-tsg-traffic-engine-numa-0-26ntr\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"c86db369-5945-4c46-9725-1cd534522a4f\"}" timestamp=1689544944068
Jul 17 03:03:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:32.404Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"9\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544940791
Jul 17 03:03:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:32.460Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_ports\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_endpoint=\"prometheus-kube-state-metrics\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", port_name=\"http\", port_number=\"8080\", port_protocol=\"TCP\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689544944068
Jul 17 03:03:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:32.472Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_created\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-delete-tsg-traffic-engine-numa-1-hxpr6\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"c86bafef-0bfe-4d1f-bcc1-adc8b0a94ad0\"}" timestamp=1689544944068
Jul 17 03:03:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:32.507Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"48\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544940939
Jul 17 03:03:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:32.521Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db2\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689544950141
Jul 17 03:03:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:32.967Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"93\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544940605
Jul 17 03:03:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:33.148Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_cpu_period\", asset=\"twa-tsgx03\", asset_id=\"95\", container=\"prometheus-node-exporter\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689544943123
Jul 17 03:03:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:33.206Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_id\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens8f1np1\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544939093
Jul 17 03:03:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:33.664Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2dlabel-TSG\\\\x2dOS\\\\x2dDATA.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689544942644
Jul 17 03:03:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:33.730Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"76\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689544940791
Jul 17 03:03:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:33.797Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_address_available\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_endpoint=\"dashboard-metrics-scraper\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689544939894
Jul 17 03:03:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:33.816Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"rpc-statd-notify.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"service\"}" timestamp=1689544947691
Jul 17 03:03:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:33.835Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-virtual-net-vetha0e03e9c.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689544947691
Jul 17 03:03:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:33.924Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_info\", asset=\"twa-tsgx05\", asset_id=\"99\", container=\"kube-state-metrics\", container_id=\"containerd://6a692a6c37775cfac2c8d862aac1e1c484c074fd3c884467f03541ec8697bb21\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", image=\"registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.5.0\", image_id=\"sha256:b781b8478c274a77dde24bf15b78795d1d564a08a1bdea43b6f38e1f366e15dd\", image_spec=\"registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.5.0\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-kube-state-metrics-6b768c7bf8-h2rv4\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"37526140-01d6-46f5-9f0b-e066a6254bac\"}" timestamp=1689544939894
Jul 17 03:03:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:33.941Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_type\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.tsg-traffic-engine-numa-0.v1\", service=\"prometheus-kube-state-metrics\", type=\"helm.sh/release.v1\"}" timestamp=1689544939894
Jul 17 03:03:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:34.033Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"81\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544940605
Jul 17 03:03:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:34.217Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link_mode\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"eno36np3\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544940605
Jul 17 03:03:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:34.279Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_free_bytes\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"shm\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/ceca645e6975d39f0681d6e1a0265568d8df6702f42db7fe01963ad704c12709/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544940605
Jul 17 03:03:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:34.296Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_changes_total\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth792cff4d\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544940605
Jul 17 03:03:38 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:38.894Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_scaling_frequency_min_hertz\", asset=\"twa-dellr740-dt003\", asset_id=\"41\", cpu=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"OLAP-Node-Exporter-twa-dellr740-dt003\", endpoint_id=\"261\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"hbase_master,clickhouse_data,kafka,gohangout,zookeeper,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689544958473
Jul 17 03:03:39 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:39.658Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_netstat_Udp_InErrors\", asset=\"twa-dellr740-dt003\", asset_id=\"41\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"OLAP-Node-Exporter-twa-dellr740-dt003\", endpoint_id=\"261\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"hbase_master,clickhouse_data,kafka,gohangout,zookeeper,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689544958473
Jul 17 03:03:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:40.234Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_scaling_frequency_hertz\", asset=\"twa-dellr740-dt003\", asset_id=\"41\", cpu=\"5\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"OLAP-Node-Exporter-twa-dellr740-dt003\", endpoint_id=\"261\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"hbase_master,clickhouse_data,kafka,gohangout,zookeeper,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689544958473
Jul 17 03:03:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:40.770Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_AppendSize_num_ops\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"WAL\"}" timestamp=1689544959756
Jul 17 03:03:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:40.962Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_restarts_total\", asset=\"twa-tsgx04\", asset_id=\"97\", container=\"prometheus-node-exporter\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-node-exporter-xrmbl\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"9f402370-116a-4e69-9fa9-b0192f6cd502\"}" timestamp=1689544960034
Jul 17 03:03:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:40.972Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_info\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"chart-values-tsg-traffic-engine-numa-1\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689544960034
Jul 17 03:03:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:40.992Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"50\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544951647
Jul 17 03:03:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:41.136Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544951647
Jul 17 03:03:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:41.148Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_vmstat_pgpgout\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544951647
Jul 17 03:03:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:41.188Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:c0-0000:c0:01.1-0000:c1:00.0-infiniband_mad-umad8.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689544955764
Jul 17 03:03:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:41.195Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_packets_total\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth3c119b31\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544951647
Jul 17 03:03:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:41.232Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db126\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689544953827
Jul 17 03:03:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:41.233Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_client_recent_max_output_buffer_bytes\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689544953827
Jul 17 03:03:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:41.249Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx04\", asset_id=\"97\", container=\"dign-server-web\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Write\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689544941316
Jul 17 03:03:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:41.294Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_numBytesCompactedCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"session_record_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"587346287ff7fe7466c7378edf2e47ec\", table=\"index_filename_session_record_hos_bucket\", type=\"index\"}" timestamp=1689544959756
Jul 17 03:03:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:41.349Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_address_assign_type\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"eno34np1\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544951647
Jul 17 03:03:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:41.375Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_load_average_10s\", asset=\"twa-tsgx04\", asset_id=\"97\", container=\"prometheus-pushgateway\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689544932967
Jul 17 03:03:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:41.389Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_status_failed\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", job_name=\"helm-install-cm-policy-local-cache\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689544960034
Jul 17 03:03:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:41.460Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_numBytesCompactedCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"firewall_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"a4729c2c59157f480f8c0f05b3fc7cb3\", table=\"index_partfile_firewall_hos_bucket\", type=\"index\"}" timestamp=1689544959756
Jul 17 03:03:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:41.483Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-random-seed.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"oneshot\", type=\"service\"}" timestamp=1689544955764
Jul 17 03:03:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:41.515Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_info\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_endpoint=\"kubernetes\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689544960034
Jul 17 03:03:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:41.536Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_10_194_1_metric_scanTime_mean\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689544959756
Jul 17 03:03:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:41.540Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_maxCompactionQueueSize\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"proxy_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"c48ff79d9c4166508d5a5bc3b3ba98c5\", table=\"index_partfile_proxy_hos_bucket\", type=\"index\"}" timestamp=1689544959756
Jul 17 03:03:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:41.553Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_10_194_1_metric_avgStoreFileAge\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689544959756
Jul 17 03:03:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:41.580Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_writeRequestCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"firewall_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"7c060b23898a6ee1a081d0e0396d1df3\", table=\"index_time_firewall_hos_bucket\", type=\"index\"}" timestamp=1689544959756
Jul 17 03:03:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:41.586Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_link_error_recovery_total\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_8\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544951647
Jul 17 03:03:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:41.592Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_storeFileCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"aaaf5d0bf083ba84a91e9eb40572d9ac\", table=\"default\", type=\"data\"}" timestamp=1689544959756
Jul 17 03:03:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:41.619Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_10_194_1_metric_tableSize\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689544959756
Jul 17 03:03:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:41.645Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"ext4\", name=\"data-var-lib-kubelet-pods-a76c8dce\\\\x2d9010\\\\x2d4080\\\\x2db63c\\\\x2d3802527a42aa-volume\\\\x2dsubpaths-config\\\\x2dpath-dign\\\\x2dserver\\\\x2ddns-1.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689544955764
Jul 17 03:03:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:41.658Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"66\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544951647
Jul 17 03:03:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:41.664Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_owner\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", owner_is_controller=\"true\", owner_kind=\"Deployment\", owner_name=\"cm-policy-local-cache-redis-master\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"cm-policy-local-cache-redis-master-69967b5879\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689544960034
Jul 17 03:03:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:41.674Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_numBytesCompactedCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"troubleshooting_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"0aad0a51cfbe947a8df6332f672987db\", table=\"index_partfile_troubleshooting_hos_bucket\", type=\"index\"}" timestamp=1689544959756
Jul 17 03:03:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:41.681Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link_mode\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens2f0np0\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544951647
Jul 17 03:03:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:41.695Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-e3512bbb32903940ad4ff09e2451f50c3dbeb1b0d7900f1831901771e5929718.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"scope\"}" timestamp=1689544955764
Jul 17 03:03:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:41.705Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_nf_conntrack_stat_ignore\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544951647
Jul 17 03:03:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:41.717Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"44\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544951647
Jul 17 03:03:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:41.724Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_compactionsFailedCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"troubleshooting_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"df9217e50c530485ade2990815b30a4b\", table=\"index_filename_troubleshooting_hos_bucket\", type=\"index\"}" timestamp=1689544959756
Jul 17 03:03:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:41.733Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"67\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544951647
Jul 17 03:03:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:41.755Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_TASK_metric_scanSize_25th_percentile\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689544959756
Jul 17 03:03:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:41.765Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"scrape_samples_scraped\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_instance=\"10.42.0.97:9273\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", vsysId=\"1\"}" timestamp=1689544953536
Jul 17 03:03:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:41.769Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_PreIndexUpdateTime_25th_percentile\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"PhoenixIndexer\"}" timestamp=1689544959756
Jul 17 03:03:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:41.804Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_CHILD_LINK_metric_incrementTime_min\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689544959756
Jul 17 03:03:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:41.821Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_BlockedFlushes_99th_percentile\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Memory\"}" timestamp=1689544959756
Jul 17 03:03:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:41.840Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_getCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"aaaf5d0bf083ba84a91e9eb40572d9ac\", table=\"default\", type=\"data\"}" timestamp=1689544959756
Jul 17 03:03:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:41.844Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_maxFlushQueueSize\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"cf8f72b463e13d4bf3fc6ce151ef1353\", table=\"rtp_hos_bucket\", type=\"data\"}" timestamp=1689544959756
Jul 17 03:03:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:41.948Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_10_194_1_metric_majorCompactionInputFileCount_max\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689544959756
Jul 17 03:03:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:42.151Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_incrementCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", namespace=\"SYSTEM\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"01b27614584382026e993c837ee3e629\", table=\"LOG\", type=\"data\"}" timestamp=1689544959756
Jul 17 03:03:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:42.808Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_packets_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"cadvisor\", interface=\"ens4f1np1\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689544958782
Jul 17 03:03:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:42.924Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_complete\", asset=\"pcap-tsgx01\", asset_id=\"71\", condition=\"unknown\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", job_name=\"helm-install-traefik\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689544955154
Jul 17 03:03:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:42.927Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_info\", asset=\"pcap-tsgx01\", asset_id=\"71\", created_by_kind=\"ReplicaSet\", created_by_name=\"coredns-79c6ccb74b\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", host_ip=\"10.10.10.161\", host_network=\"false\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", node=\"localhost\", nz_agent_id=\"3\", pod=\"coredns-79c6ccb74b-rpzl7\", pod_ip=\"10.42.0.54\", priority_class=\"system-cluster-critical\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"be245aed-e373-405f-8d79-93c5764d578b\"}" timestamp=1689544955154
Jul 17 03:03:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:43.074Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_ulimits_soft\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", ulimit=\"max_open_files\"}" timestamp=1689544959734
Jul 17 03:03:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:43.715Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_packets_dropped_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"cadvisor\", interface=\"ens2f0np0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689544958782
Jul 17 03:03:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:43.793Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_packets_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"cadvisor\", interface=\"virtio_dign_c\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689544958782
Jul 17 03:03:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:43.828Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_metadata_generation\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"mrzcpd-exporter\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689544955154
Jul 17 03:03:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:43.876Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_annotations\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"kubernetes-dashboard\", uid=\"42530dde-0d12-4fdb-830d-4ce944558e33\"}" timestamp=1689544955154
Jul 17 03:03:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:43.974Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", cpu=\"86\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544956242
Jul 17 03:03:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:43.989Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_size_bytes\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"shm\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/e12645fd77e0267b7e4793e719ced23b1c706f0f70bf9a2b3489baaad542935d/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544956242
Jul 17 03:03:44 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:44.004Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"initrd-parse-etc.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689544960671
Jul 17 03:03:44 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:44.066Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"iscsi-shutdown.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689544960671
Jul 17 03:03:44 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:44.179Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_timex_offset_seconds\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544956242
Jul 17 03:03:44 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:44.237Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"auth-rpcgss-module.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"service\"}" timestamp=1689544960671
Jul 17 03:03:44 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:44.241Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", phase=\"Unknown\", pod=\"prometheus-pushgateway-bd476bcf9-scrs2\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"1d9c1047-30bc-4de8-bd00-07e7917cd6b7\"}" timestamp=1689544955154
Jul 17 03:03:44 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:44.268Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_labels\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-1-firewall-786576fb8\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689544955154
Jul 17 03:03:44 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:44.297Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_annotations\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", uid=\"608dc4ed-eac0-4c84-a522-931a51275eca\"}" timestamp=1689544955154
Jul 17 03:03:44 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:44.316Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_constraint_errors_transmitted_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"mlx5_6\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544956242
Jul 17 03:03:47 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:47.991Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"clickhouse_background_move_pool_task\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Clickhouse-msh-dellr740-dt001\", endpoint_id=\"79\", module=\"NC-Clickhouse\", module_id=\"29\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689544967155
Jul 17 03:03:48 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:48.060Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"clickhouse_network_receive_packets_br_851e3f3d436f\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Clickhouse-msh-dellr740-dt001\", endpoint_id=\"79\", module=\"NC-Clickhouse\", module_id=\"29\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689544967155
Jul 17 03:03:48 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:48.618Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_193_1_metric_scanSize_25th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689544967438
Jul 17 03:03:48 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:48.711Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_session_record_hos_bucket_metric_flushMemstoreSize_75th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689544967438
Jul 17 03:03:48 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:48.849Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_deleteCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"session_record_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"5e4a8e5af9d1dd712c1dd2749d210276\", table=\"index_partfile_session_record_hos_bucket\", type=\"index\"}" timestamp=1689544967438
Jul 17 03:03:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:48.990Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_194_3_metric_compactionTime_99th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689544967438
Jul 17 03:03:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:49.402Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Bulkload_25th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Server\"}" timestamp=1689544967438
Jul 17 03:03:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:49.897Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_terminated\", asset=\"twa-tsgx08\", asset_id=\"105\", container=\"kubernetes-dashboard\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", pod=\"kubernetes-dashboard-66dd5b456-5x4qv\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"a0e438ae-62dd-4767-974d-b0821c7c31f3\"}" timestamp=1689544961568
Jul 17 03:03:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:49.940Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_info\", asset=\"twa-tsgx08\", asset_id=\"105\", configmap=\"telegraf-tsg-traffic-engine-numa-1\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689544961568
Jul 17 03:03:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:49.954Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"49\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544966709
Jul 17 03:03:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:49.983Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_default_metric_scanSize_75th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689544967438
Jul 17 03:03:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:49.993Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"kube-state-metrics\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", failure_type=\"pgmajfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"container\"}" timestamp=1689544953862
Jul 17 03:03:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:50.016Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_cpu_period\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"lb-tcp-80\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689544959050
Jul 17 03:03:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:50.018Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_session_record_hos_bucket_metric_majorCompactionOutputFileCount_98th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689544967438
Jul 17 03:03:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:50.037Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_4_metric_getTime_min\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689544967438
Jul 17 03:03:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:50.093Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-6b43516fda13e768eb516cdc1c827c68d31eccf0f9e840e098fbeff3ac8f9cbc-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689544961057
Jul 17 03:03:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:50.106Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"tsg_master_activing\", app_name=\"tsg_master\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"traffic-engine\", line_name=\"fail\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", table_name=\"ctrl_stat\", vsysId=\"1\"}" timestamp=1689544962447
Jul 17 03:03:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:50.125Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_194_4_metric_flushMemstoreSize_25th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689544967438
Jul 17 03:03:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:50.128Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"tsg-clixon-backend.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689544966554
Jul 17 03:03:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:50.148Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-sda3.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689544966554
Jul 17 03:03:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:50.187Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_3_metric_putBatchTime_90th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689544967438
Jul 17 03:03:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:50.190Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_readRequestCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"firewall_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"569c4ca2944511d5d885d0d8a8bc9016\", table=\"index_partfile_firewall_hos_bucket\", type=\"index\"}" timestamp=1689544967438
Jul 17 03:03:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:50.216Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_numBytesCompactedCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"session_record_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"5f0899b44135b34982bf8c9cfd51c7b4\", table=\"index_time_session_record_hos_bucket\", type=\"index\"}" timestamp=1689544967438
Jul 17 03:03:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:50.221Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_minStoreFileAge\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"assessment_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"3fbd8de5bb4d656118dcc5163fc3bd6d\", table=\"index_partfile_assessment_hos_bucket\", type=\"index\"}" timestamp=1689544967438
Jul 17 03:03:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:50.231Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_default_metric_compactionTime_95th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689544967438
Jul 17 03:03:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:50.285Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"81\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544966709
Jul 17 03:03:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:50.289Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-20025742b4f87b7ff1b63ca328d232a8493992ad6cc32f04292c37fe5dc27577-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689544966554
Jul 17 03:03:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:50.298Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_master_link_up\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", master_host=\"192.168.20.157\", master_port=\"7112\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689544967556
Jul 17 03:03:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:50.336Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_2_metric_putBatchTime_99_9th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689544967438
Jul 17 03:03:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:50.453Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx08\", asset_id=\"105\", container=\"dign-client\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Sync\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689544965043
Jul 17 03:03:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:50.471Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db19\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689544967556
Jul 17 03:03:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:50.560Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_tsg_galaxy_table_relation_user_teid_metric_tableWriteQueryPerSecond_count\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689544967438
Jul 17 03:03:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:50.609Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_commands_failed_calls_total\", asset=\"twa-tsgx10\", asset_id=\"109\", cmd=\"select\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689544966625
Jul 17 03:03:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:50.702Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_name_assign_type\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"eno34np1\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544961797
Jul 17 03:03:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:50.719Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-3ee8ed2fb8aa3aa8481c872a5e74e8ae17eb1b93bdf54f0ab2f26ef19f83afc0.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"scope\"}" timestamp=1689544961057
Jul 17 03:03:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:50.825Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db195\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689544966625
Jul 17 03:03:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:50.828Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_ulimits_soft\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"kube-state-metrics\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", ulimit=\"max_open_files\"}" timestamp=1689544953862
Jul 17 03:03:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:50.831Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-6247313a5016c06851403e117a37ec4c0add7ca2825c21d314257bbe145bf711.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"scope\"}" timestamp=1689544961057
Jul 17 03:03:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:50.852Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-60382aa1a33ac5e782171c50ff4d96707752395ccde39c1e398771bd606f14ff-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689544961057
Jul 17 03:03:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:50.870Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"29\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544966709
Jul 17 03:03:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:50.888Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_annotations\", asset=\"twa-tsgx08\", asset_id=\"105\", configmap=\"sce-tsg-traffic-engine-numa-0\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689544961568
Jul 17 03:03:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:50.907Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_memory_PageTables_bytes\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544966709
Jul 17 03:03:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:50.912Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx10\", asset_id=\"109\", cpu=\"86\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544961797
Jul 17 03:03:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:50.923Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_changes_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"lo\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544966709
Jul 17 03:03:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:50.925Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"ftx_bits_total\", app=\"sapp4-1\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"nf_1_fw\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"mrzcpd-exporter\", serviceFunctionIndex=\"1\"}" timestamp=1689544955675
Jul 17 03:03:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:50.968Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_files\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"shm\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/60382aa1a33ac5e782171c50ff4d96707752395ccde39c1e398771bd606f14ff/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544961797
Jul 17 03:03:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:50.976Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"46\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544966709
Jul 17 03:03:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:50.995Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_state_id\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_3\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544966709
Jul 17 03:03:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:51.014Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"twa-tsgx10\", asset_id=\"109\", cpu=\"86\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544961797
Jul 17 03:03:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:51.034Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-7b43283a\\\\x2d781d\\\\x2d45c1\\\\x2d8760\\\\x2de8b3248f8fd2-volume\\\\x2dsubpaths-config\\\\x2dpath-dign\\\\x2dserver\\\\x2ddns-1.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689544961057
Jul 17 03:03:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:51.057Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"slice\"}" timestamp=1689544961057
Jul 17 03:03:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:51.070Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_client_recent_max_input_buffer_bytes\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689544966625
Jul 17 03:03:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:51.311Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544966709
Jul 17 03:03:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:52.697Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", cpu=\"108\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544969337
Jul 17 03:03:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:52.788Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", cpu=\"1\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544969337
Jul 17 03:03:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:52.817Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"tmpfs\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/var/lib/kubelet/pods/22030e9c-e81d-45d2-a48a-b2673ca30200/volumes/kubernetes.io~projected/kube-api-access-54l4n\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544969337
Jul 17 03:03:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:52.884Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_limit_bytes\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/sys/fs/cgroup\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689544969298
Jul 17 03:03:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:52.911Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_complete\", asset=\"pcap-tsgx11\", asset_id=\"115\", condition=\"unknown\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", job_name=\"helm-install-configmap-reloader\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689544964478
Jul 17 03:03:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:53.232Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_repl_backlog_history_bytes\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689544970108
Jul 17 03:03:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:53.325Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", container=\"configmap-reloader\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Write\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689544955636
Jul 17 03:03:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:53.368Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2dpath-pci\\\\x2d0000:81:00.0\\\\x2dscsi\\\\x2d0:2:0:0\\\\x2dpart3.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689544965496
Jul 17 03:03:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:53.381Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"clickhouse_os_guest_nice_time\", asset=\"pcap-dellr740-dt003\", asset_id=\"35\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"NC-Clickhouse-pcap-dellr740-dt003\", endpoint_id=\"85\", module=\"NC-Clickhouse\", module_id=\"29\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689544973231
Jul 17 03:03:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:53.389Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-virtual-net-br_dign_s.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689544972135
Jul 17 03:03:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:53.407Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"clickhouse_os_nice_time_cpu38\", asset=\"pcap-dellr740-dt003\", asset_id=\"35\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"NC-Clickhouse-pcap-dellr740-dt003\", endpoint_id=\"85\", module=\"NC-Clickhouse\", module_id=\"29\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689544973231
Jul 17 03:03:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:53.412Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sockets.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"target\"}" timestamp=1689544965496
Jul 17 03:03:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:53.470Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_labels\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", uid=\"397e64c9-ec74-4164-840f-f822f57cde52\"}" timestamp=1689544964478
Jul 17 03:03:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:53.488Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", cpu=\"62\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544969337
Jul 17 03:03:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:53.552Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"48\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689544963391
Jul 17 03:03:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:53.565Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-hwdb-update.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"service\"}" timestamp=1689544971179
Jul 17 03:03:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:53.592Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-ttyS1.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689544965496
Jul 17 03:03:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:53.669Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_namespace_created\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689544964478
Jul 17 03:03:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:53.687Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_udp_queues\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", ip=\"v6\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", queue=\"rx\", service=\"prometheus-node-exporter\"}" timestamp=1689544963391
Jul 17 03:03:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:53.689Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_name_assign_type\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"vethcf8870d8\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544959409
Jul 17 03:03:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:53.705Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"msh-tsgx12\", asset_id=\"113\", condition=\"Progressing\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"traefik\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"true\"}" timestamp=1689544968663
Jul 17 03:03:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:53.723Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"bond0\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544968328
Jul 17 03:03:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:53.728Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_scrape_collector_success\", asset=\"pcap-tsgx07\", asset_id=\"85\", collector=\"xfs\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544959409
Jul 17 03:03:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:53.729Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_dormant\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"bond0\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544963391
Jul 17 03:03:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:53.748Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_spec_type\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"metrics-server\", type=\"ClusterIP\", uid=\"0e6c4ffe-1d33-4290-bc90-2ff596585379\"}" timestamp=1689544968663
Jul 17 03:03:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:53.752Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2dpartlabel-ONIE\\\\x2dBOOT.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689544960884
Jul 17 03:03:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:53.768Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"ext4\", name=\"mnt-onie\\\\x2dboot.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689544971179
Jul 17 03:03:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:53.790Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cpu=\"37\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544959409
Jul 17 03:03:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:53.816Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cpu=\"20\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544959409
Jul 17 03:03:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:53.820Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_name_assign_type\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"br_dign_s\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544968328
Jul 17 03:03:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:53.862Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"getty-pre.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"target\"}" timestamp=1689544971179
Jul 17 03:03:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:53.917Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"pcap-tsgx07\", asset_id=\"85\", condition=\"Progressing\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"prometheus-kube-state-metrics\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"unknown\"}" timestamp=1689544962058
Jul 17 03:03:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:53.929Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"machine_scrape_error\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689544970608
Jul 17 03:03:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:53.954Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Tcp_Link_New\", app_name=\"sapp4\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", vsysId=\"1\"}" timestamp=1689544967223
Jul 17 03:03:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:53.956Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_dmi_info\", asset=\"pcap-tsgx11\", asset_id=\"115\", bios_date=\"07/08/2020\", bios_release=\"1.4\", bios_version=\"1.4.78\", board_name=\"0590KW\", board_version=\"A01\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", product_sku=\"SKU=NotProvided;ModelName=\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544969337
Jul 17 03:03:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:53.964Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx12\", asset_id=\"113\", cpu=\"109\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544963391
Jul 17 03:03:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:53.982Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_fifo_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"cnibr0\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544959409
Jul 17 03:03:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:53.989Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_packets_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"vethfd1158fa\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544959409
Jul 17 03:03:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:54.008Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_multicast_total\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens4f0np0\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544968328
Jul 17 03:03:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:54.029Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_carrier_total\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"vethb1d236b8\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544968328
Jul 17 03:03:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:54.043Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db65\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689544970108
Jul 17 03:03:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:54.058Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-e470866ae69b810e02897446e318594a536c3974a04b4d7c4d9b090557f53bef.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"scope\"}" timestamp=1689544960884
Jul 17 03:03:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:54.061Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"rpcbind.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"notify\", type=\"service\"}" timestamp=1689544960884
Jul 17 03:03:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:54.073Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_hwmon_temp_celsius\", asset=\"pcap-tsgx11\", asset_id=\"115\", chip=\"pci0000:00_0000:00:18_3\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", sensor=\"temp3\", service=\"prometheus-node-exporter\"}" timestamp=1689544969337
Jul 17 03:03:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:54.081Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_mspan_inuse_bytes\", asset=\"pcap-dellr740-dt002\", asset_id=\"33\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"NC-Gohangout-pcap-dellr740-dt002\", endpoint_id=\"173\", module=\"DC-Gohangout\", module_id=\"35\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689544972666
Jul 17 03:03:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:54.091Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_annotations\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", job_name=\"helm-install-mrzcpd-exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689544962058
Jul 17 03:03:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:54.106Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_reservation_limit_bytes\", asset=\"pcap-tsgx07\", asset_id=\"85\", container=\"ipmi-exporter\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689544970608
Jul 17 03:03:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:54.126Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_created\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_endpoint=\"cm-policy-local-cache-redis-metrics\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689544962058
Jul 17 03:03:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:54.189Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_down_changes_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth7bb95ccb\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544969337
Jul 17 03:03:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:54.204Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db104\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689544959367
Jul 17 03:03:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:54.217Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"tmpfs\", name=\"run-k3s-containerd-io.containerd.grpc.v1.cri-sandboxes-5b2e5aa8455ace000f11eec393f2841e58cba3391f46a8ba95d2a64f4eb7a648-shm.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689544960884
Jul 17 03:03:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:54.218Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_packets_total\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth321ff728\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544963391
Jul 17 03:03:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:54.241Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"clickhouse_compile_expressions_microseconds_total\", asset=\"pcap-dellr740-dt003\", asset_id=\"35\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"NC-Clickhouse-pcap-dellr740-dt003\", endpoint_id=\"85\", module=\"NC-Clickhouse\", module_id=\"29\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689544973231
Jul 17 03:03:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:54.245Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-5a7f8ba1f2ccbef011022ca1c740ddd7990b3f9e89f3095ee187322a62deec8f.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"scope\"}" timestamp=1689544971179
Jul 17 03:03:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:54.256Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_netstat_Udp_NoPorts\", asset=\"pcap-dellr740-dt002\", asset_id=\"33\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"NC-Gohangout-pcap-dellr740-dt002\", endpoint_id=\"173\", module=\"DC-Gohangout\", module_id=\"35\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689544972666
Jul 17 03:03:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:54.275Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_packets_received_total\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_9\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544963391
Jul 17 03:03:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:54.303Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-ens7f0np0.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689544960884
Jul 17 03:03:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:54.320Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx12\", asset_id=\"113\", cpu=\"108\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544963391
Jul 17 03:03:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:54.369Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"abrt-ccpp.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689544960884
Jul 17 03:03:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:54.386Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2duuid-b5d92871\\\\x2d8e4c\\\\x2d442f\\\\x2db3e0\\\\x2d215efb7cc0fe.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689544960884
Jul 17 03:03:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:54.401Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_created\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"metrics-server-54dc485875\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689544968663
Jul 17 03:03:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:54.417Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"iscsi-onboot.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"service\"}" timestamp=1689544971179
Jul 17 03:03:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:54.420Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"var-lib-kubelet-pods-c62cc0c7\\\\x2db670\\\\x2d44ac\\\\x2dbe9c\\\\x2df9cc027d18ec-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2dwjd8c.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689544960884
Jul 17 03:03:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:54.556Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cpu=\"79\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544959409
Jul 17 03:03:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:54.570Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_netstat_TcpExt_SyncookiesFailed\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544959409
Jul 17 03:03:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:54.573Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_virtual_memory_bytes\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"rtkit-daemon.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689544960884
Jul 17 03:03:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:54.586Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-bc2cea3fa2f86df669e27dd1e22675e83ceaf976ae3629739a3f98bd2d660c73.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"scope\"}" timestamp=1689544960884
Jul 17 03:03:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:54.651Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dracut-pre-trigger.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"service\"}" timestamp=1689544960884
Jul 17 03:03:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:54.765Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"emergency.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"target\"}" timestamp=1689544960884
Jul 17 03:03:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:54.790Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_mtu_bytes\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"vethc318cddc\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544959409
Jul 17 03:03:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:54.805Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"openibd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689544960884
Jul 17 03:03:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:56.622Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"114\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544965883
Jul 17 03:03:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:57.151Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_metadata_resource_version\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"kubernetes-dashboard-certs\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689544973389
Jul 17 03:03:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:57.210Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544965883
Jul 17 03:03:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:57.321Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"127\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544965883
Jul 17 03:03:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:57.390Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_down_changes_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"virtio_dign_s\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544965883
Jul 17 03:03:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:57.419Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db158\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689544966475
Jul 17 03:03:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:57.493Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2did-scsi\\\\x2dSDELL_PERC_H345_Adp_0065e1fae1717c2c2c00578284f0a7ce\\\\x2dpart3.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689544963372
Jul 17 03:03:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:57.534Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"mrzcpd_hugepages_setup.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"service\"}" timestamp=1689544963372
Jul 17 03:03:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:57.635Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"102\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544965883
Jul 17 03:03:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:57.651Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"mdmonitor.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689544963372
Jul 17 03:03:58 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:58.468Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_edac_csrow_uncorrectable_errors_total\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", controller=\"2\", csrow=\"unknown\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"OLAP-Node-Exporter-pcap-dellr740-dt001\", endpoint_id=\"251\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689544977557
Jul 17 03:03:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:59.788Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"91\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544969083
Jul 17 03:03:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:59.858Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-hwdb-update.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689544964879
Jul 17 03:03:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:03:59.919Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"tmpfs\", name=\"var-lib-kubelet-pods-fb6fed9b\\\\x2d0334\\\\x2d48d9\\\\x2db65e\\\\x2d577f418342fd-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2dpm4b5.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689544964879
Jul 17 03:04:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:00.258Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_threads_max\", asset=\"msh-tsgx01\", asset_id=\"51\", container=\"prometheus-pushgateway\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689544958240
Jul 17 03:04:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:00.294Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"4\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544969083
Jul 17 03:04:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:00.310Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"cadvisor\", failure_type=\"pgmajfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"container\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689544972947
Jul 17 03:04:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:00.329Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_fifo_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth844980e5\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544969083
Jul 17 03:04:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:00.402Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"109\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689544969083
Jul 17 03:04:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:00.509Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db46\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689544965071
Jul 17 03:04:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:00.759Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_compressed_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"eno2\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544969083
Jul 17 03:04:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:00.836Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"108\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544969083
Jul 17 03:04:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:00.844Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"getty@tty1.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689544964879
Jul 17 03:04:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:00.908Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_cache\", asset=\"msh-tsgx01\", asset_id=\"51\", container=\"coredns\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689544967503
Jul 17 03:04:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:02.511Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", pod=\"kubernetes-dashboard-66dd5b456-krsfr\", project=\"TSG-OS\", project_id=\"7\", reason=\"NodeLost\", service=\"prometheus-kube-state-metrics\", uid=\"87913277-0e7d-4545-8517-0068668d7130\"}" timestamp=1689544971412
Jul 17 03:04:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:02.867Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx02\", asset_id=\"63\", cpu=\"28\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544978318
Jul 17 03:04:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:03.017Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_queue_length\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth2ff062e3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544978318
Jul 17 03:04:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:03.038Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"local-fs-pre.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"target\"}" timestamp=1689544968447
Jul 17 03:04:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:03.133Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_reads_merged_total\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/run\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689544965217
Jul 17 03:04:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:03.174Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_netstat_Udp_InDatagrams\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-dt001\", endpoint_id=\"225\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689544982988
Jul 17 03:04:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:03.187Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"net_conntrack_dialer_conn_closed_total\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", dialer_name=\"blackbox_exporter_metrics\", instance=\"10.10.20.159:10090\", type=\"global\"}" timestamp=1689544982989
Jul 17 03:04:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:03.270Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_bytes_total\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"vethad44a482\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544978318
Jul 17 03:04:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:03.285Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-7336ebc13f6a18441b091a169d603ade439ceb80fc0ae83a8ac2dc7d3ffd4569.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"scope\"}" timestamp=1689544968447
Jul 17 03:04:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:03.340Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"net_conntrack_dialer_conn_attempted_total\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", dialer_name=\"45\", instance=\"10.10.20.159:10090\", type=\"global\"}" timestamp=1689544982989
Jul 17 03:04:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:03.362Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"net_conntrack_dialer_conn_failed_total\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", dialer_name=\"107\", instance=\"10.10.20.159:10090\", reason=\"timeout\", type=\"global\"}" timestamp=1689544982989
Jul 17 03:04:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:03.486Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_labels\", asset=\"msh-tsgx02\", asset_id=\"63\", configmap=\"multus-cni-config\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689544971412
Jul 17 03:04:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:03.532Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_receive_remote_physical_errors_total\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_1\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544978318
Jul 17 03:04:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:03.591Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_persistentvolume_info\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", persistentvolume=\"pvc-510dae2f-c3b6-4fc8-8fb4-b442f2d2e24c\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", storageclass=\"local-path\"}" timestamp=1689544971412
Jul 17 03:04:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:03.627Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_running\", asset=\"msh-tsgx02\", asset_id=\"63\", container=\"helm\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-tsg-traffic-engine-numa-1-lhncj\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"35ba78e1-f4f4-4fdd-97e3-b2303cb918ac\"}" timestamp=1689544971412
Jul 17 03:04:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:03.649Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"msh-tsgx02\", asset_id=\"63\", cpu=\"59\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544978318
Jul 17 03:04:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:03.692Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_io_time_seconds_total\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"overlay_0-47\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689544965217
Jul 17 03:04:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:03.715Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_working_set_bytes\", asset=\"msh-tsgx02\", asset_id=\"63\", container=\"kube-state-metrics\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689544961795
Jul 17 03:04:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:03.758Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-infiniband-umad6.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689544968447
Jul 17 03:04:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:03.765Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_created\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"cm-policy-local-cache-redis-master-69967b5879\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689544971412
Jul 17 03:04:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:03.767Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db192\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689544969269
Jul 17 03:04:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:03.862Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_hwmon_temp_celsius\", asset=\"pcap-dellr740-dt002\", asset_id=\"33\", chip=\"platform_coretemp_0\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"OLAP-Node-Exporter-pcap-dellr740-dt002\", endpoint_id=\"253\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\", sensor=\"temp6\"}" timestamp=1689544982497
Jul 17 03:04:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:03.874Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dracut-pre-mount.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"service\"}" timestamp=1689544968447
Jul 17 03:04:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:03.879Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"prometheus_target_sync_length_seconds\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", instance=\"10.10.20.159:10090\", quantile=\"0.01\", scrape_job=\"285\", type=\"global\"}" timestamp=1689544982989
Jul 17 03:04:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:03.902Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_scrape_collector_duration_seconds\", asset=\"pcap-dellr740-dt002\", asset_id=\"33\", collector=\"fibrechannel\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"OLAP-Node-Exporter-pcap-dellr740-dt002\", endpoint_id=\"253\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689544982497
Jul 17 03:04:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:03.954Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"prometheus_target_sync_length_seconds_sum\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", instance=\"10.10.20.159:10090\", scrape_job=\"391\", type=\"global\"}" timestamp=1689544982989
Jul 17 03:04:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:04.000Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx02\", asset_id=\"63\", container=\"dign-client\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Sync\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689544957524
Jul 17 03:04:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:04.380Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_frequency_max_hertz\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", cpu=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-dt001\", endpoint_id=\"225\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689544982988
Jul 17 03:04:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:04.452Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"net_conntrack_dialer_conn_failed_total\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", dialer_name=\"387\", instance=\"10.10.20.159:10090\", reason=\"resolution\", type=\"global\"}" timestamp=1689544982989
Jul 17 03:04:05 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:05.786Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"alert_rule_eval_state\", id=\"100087\"}" timestamp=1689545024819
Jul 17 03:04:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:06.617Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", cpu=\"40\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Spark-msh-dellr740-olap003\", endpoint_id=\"13\", mode=\"nice\", module=\"NC-Spark\", module_id=\"47\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689544986091
Jul 17 03:04:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:06.754Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_frequency_max_hertz\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", cpu=\"38\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Spark-msh-dellr740-olap003\", endpoint_id=\"13\", module=\"NC-Spark\", module_id=\"47\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689544986091
Jul 17 03:04:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:06.798Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_sockstat_TCP_orphan\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Spark-msh-dellr740-olap003\", endpoint_id=\"13\", module=\"NC-Spark\", module_id=\"47\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689544986091
Jul 17 03:04:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:06.817Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_errs_total\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens3f0np0\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544983910
Jul 17 03:04:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:06.874Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-eno34np1.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689544973582
Jul 17 03:04:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:06.995Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_fifo_total\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth2d0ed10f\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544983910
Jul 17 03:04:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:07.066Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_gc_duration_seconds\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Spark-msh-dellr740-olap003\", endpoint_id=\"13\", module=\"NC-Spark\", module_id=\"47\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", quantile=\"0.75\"}" timestamp=1689544986091
Jul 17 03:04:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:07.067Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_changes_total\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"eno33np0\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544983910
Jul 17 03:04:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:07.183Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"druid_compact_waiting_task_num\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Spark-msh-dellr740-olap003\", endpoint_id=\"13\", module=\"NC-Spark\", module_id=\"47\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689544986091
Jul 17 03:04:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:08.091Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_hwmon_temp_max_celsius\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", chip=\"platform_coretemp_0\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Spark-msh-dellr740-olap003\", endpoint_id=\"13\", module=\"NC-Spark\", module_id=\"47\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sensor=\"temp13\"}" timestamp=1689544986091
Jul 17 03:04:09 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:09.823Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_getCount\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"b939a560a303d7de0c3652cd83cc8562\", table=\"proxy_hos_bucket\", type=\"data\"}" timestamp=1689544988804
Jul 17 03:04:09 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:09.856Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_blockCacheIntermediateIndexMissCount\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Server\"}" timestamp=1689544988804
Jul 17 03:04:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:10.145Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Put_98th_percentile\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Server\"}" timestamp=1689544988804
Jul 17 03:04:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:10.214Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_minStoreFileAge\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"rtp_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"39c380d9031a49d2b0d03da22a2f14b9\", table=\"index_time_rtp_hos_bucket\", type=\"index\"}" timestamp=1689544988804
Jul 17 03:04:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:10.621Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_compactionsFailedCount\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"proxy_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"837d0bb3c8b8aa4665a4ace2b6e2c6ed\", table=\"index_filename_proxy_hos_bucket\", type=\"index\"}" timestamp=1689544988804
Jul 17 03:04:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:10.716Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_numBytesCompactedCount\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"firewall_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"335727b6b83d9b052005b2861958972d\", table=\"index_time_firewall_hos_bucket\", type=\"index\"}" timestamp=1689544988804
Jul 17 03:04:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:11.453Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_incrementCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"proxy_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"0c8a74488b2d13832aa7ff1474520d22\", table=\"index_filename_proxy_hos_bucket\", type=\"index\"}" timestamp=1689544990147
Jul 17 03:04:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:11.473Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_maxCompactionQueueSize\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"proxy_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"8340614eac320cffae52bde3cbcba97e\", table=\"index_partfile_proxy_hos_bucket\", type=\"index\"}" timestamp=1689544988804
Jul 17 03:04:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:11.956Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_memStoreSize\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"firewall_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"9ce36b514c74bdc9091ef6f014dba891\", table=\"index_partfile_firewall_hos_bucket\", type=\"index\"}" timestamp=1689544988804
Jul 17 03:04:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:12.072Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_STATS_metric_compactionInputFileCount_min\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689544988804
Jul 17 03:04:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:12.200Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_FsPReadTime_25th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"IO\"}" timestamp=1689544990147
Jul 17 03:04:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:12.246Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_frequency_min_hertz\", asset=\"twa-dellr740-dt002\", asset_id=\"39\", cpu=\"42\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"NC-Gohangout-twa-dellr740-dt002\", endpoint_id=\"179\", module=\"DC-Gohangout\", module_id=\"35\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689544991525
Jul 17 03:04:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:12.316Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_193_1_metric_compactedInputBytes\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689544990147
Jul 17 03:04:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:12.440Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_avgStoreFileAge\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", namespace=\"SYSTEM\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"71f969630421eb2700553877cf2f18d9\", table=\"LOG\", type=\"data\"}" timestamp=1689544990147
Jul 17 03:04:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:12.651Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_compactionsCompletedCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"knowledge_base_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"13a4e6025bf464f6e219ed4362f86f50\", table=\"index_time_knowledge_base_hos_bucket\", type=\"index\"}" timestamp=1689544990147
Jul 17 03:04:13 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:13.003Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"twa-dellr740-dt002\", asset_id=\"39\", cpu=\"22\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"OLAP-Node-Exporter-twa-dellr740-dt002\", endpoint_id=\"259\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689544992055
Jul 17 03:04:13 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:13.948Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"twa-dellr740-dt002\", asset_id=\"39\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"OLAP-Node-Exporter-twa-dellr740-dt002\", endpoint_id=\"259\", module=\"OLAP-Node-Exporter\", module_id=\"51\", name=\"30\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\", type=\"Processor\"}" timestamp=1689544992055
Jul 17 03:04:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:15.011Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_scaling_frequency_max_hertz\", asset=\"twa-dellr740-dt002\", asset_id=\"39\", cpu=\"8\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"OLAP-Node-Exporter-twa-dellr740-dt002\", endpoint_id=\"259\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689544992055
Jul 17 03:04:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:15.612Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx07\", asset_id=\"103\", container=\"firewall\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Read\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689544978730
Jul 17 03:04:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:15.690Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_bytes_total\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"cadvisor\", interface=\"ens4f3np3\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689544989130
Jul 17 03:04:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:15.762Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_start_time_seconds\", asset=\"twa-tsgx07\", asset_id=\"103\", container=\"dign-client\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689544989150
Jul 17 03:04:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:15.781Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_info\", asset=\"twa-tsgx07\", asset_id=\"103\", configmap=\"extension-apiserver-authentication\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689544988076
Jul 17 03:04:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:15.806Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_terminated\", asset=\"twa-tsgx07\", asset_id=\"103\", container=\"prometheus-node-exporter\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-node-exporter-pfqsj\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"85ce1d65-33c2-41b7-81ab-b25e87a182c7\"}" timestamp=1689544988076
Jul 17 03:04:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:15.869Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_created\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-pushgateway\", uid=\"a575e370-a56c-41b6-8e47-5faef30c0e8a\"}" timestamp=1689544988076
Jul 17 03:04:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:15.996Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_errors_total\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"cadvisor\", interface=\"ens2f0np0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689544989130
Jul 17 03:04:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:16.052Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_labels\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", job_name=\"helm-install-systemd-exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689544988076
Jul 17 03:04:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:16.094Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_state_id\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_4\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544981230
Jul 17 03:04:17 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:17.393Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_errs_total\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens3f1np1\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544981230
Jul 17 03:04:17 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:17.903Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"cadvisor_version_info\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"cadvisor\", kernelVersion=\"5.17.15-1.el8.x86_64\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", osVersion=\"TSG-OS v23.07-rc1-079e964 (TSGXNXR620G40R01P0906)\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689544989150
Jul 17 03:04:17 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:17.906Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"54\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544981230
Jul 17 03:04:17 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:17.955Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_info\", asset=\"twa-tsgx07\", asset_id=\"103\", configmap=\"sce-tsg-traffic-engine-numa-0\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689544988076
Jul 17 03:04:18 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:18.016Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_mtu_bytes\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens8f1np1\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544981230
Jul 17 03:04:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:19.659Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:80-0000:80:01.1-0000:81:00.0-host0-target0:2:0-0:2:0:0-block-sda.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689544984356
Jul 17 03:04:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:20.861Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Pending\", pod=\"helm-install-traefik-b7tft\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"f8334eab-55b4-4187-9700-4ad2897e2d91\"}" timestamp=1689544993828
Jul 17 03:04:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:20.863Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"exporter-systemd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689544990657
Jul 17 03:04:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:20.961Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_spec_replicas\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-1-firewall-6b6bf4679c\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689544993828
Jul 17 03:04:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:20.962Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_writes_merged_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/sys/fs/cgroup\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689544995245
Jul 17 03:04:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:21.002Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dnf-makecache.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689544990657
Jul 17 03:04:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:21.499Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"82\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544988263
Jul 17 03:04:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:21.564Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_init_container_resource_requests\", asset=\"pcap-tsgx10\", asset_id=\"89\", container=\"install-multus-binary\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", node=\"localhost\", nz_agent_id=\"3\", pod=\"kube-multus-ds-sqrb7\", project=\"TSG-OS\", project_id=\"7\", resource=\"cpu\", service=\"prometheus-kube-state-metrics\", uid=\"ee8afe42-b7fb-4840-a9ee-3d0e7212c356\", unit=\"core\"}" timestamp=1689544993828
Jul 17 03:04:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:21.598Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_labels\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"traefik-64f55bb67d-v57x6\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"eb9c8310-844c-4b72-9e0c-d86e20973848\"}" timestamp=1689544993828
Jul 17 03:04:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:21.670Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"atd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"service\"}" timestamp=1689544990657
Jul 17 03:04:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:21.849Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"twa-tsgx06\", asset_id=\"101\", container=\"lb-tcp-443\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"sleeping\"}" timestamp=1689544966701
Jul 17 03:04:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:22.215Z caller=dedupe.go:112 component=remote level=error remote_name=52db6f url=http://10.10.10.159:10090/nz-agent/prometheus/proxy/api/v1/write msg="non-recoverable error" count=1000 exemplarCount=0 err="server returned HTTP status 400 : out of order sample"
Jul 17 03:04:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:22.272Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_device_id\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth814db6c2\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544997612
Jul 17 03:04:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:22.298Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_limit_bytes\", asset=\"pcap-tsgx10\", asset_id=\"89\", container=\"telegraf\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689544995735
Jul 17 03:04:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:22.333Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-kube-state-metrics-6b768c7bf8-xsfwc\", project=\"TSG-OS\", project_id=\"7\", reason=\"Evicted\", service=\"prometheus-kube-state-metrics\", uid=\"10741627-0ed0-416d-ac50-5a8214dc1db7\"}" timestamp=1689544993828
Jul 17 03:04:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:22.418Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_writes_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda5\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689544995245
Jul 17 03:04:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:24.074Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2did-scsi\\\\x2dSDELL_PERC_H345_Adp_00c4cde91457202c2c00638284f0a7ce\\\\x2dpart3.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689544998416
Jul 17 03:04:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:24.174Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"ipmi-exporter-6f8464dd94-8x6qr\", project=\"TSG-OS\", project_id=\"7\", reason=\"Evicted\", service=\"prometheus-kube-state-metrics\", uid=\"b7741d71-12eb-4a84-9915-4d0db07d0a79\"}" timestamp=1689544991122
Jul 17 03:04:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:24.221Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx09\", asset_id=\"55\", cpu=\"126\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544994790
Jul 17 03:04:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:24.358Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"pcap-tsgx12\", asset_id=\"117\", condition=\"unknown\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-prometheus-glllf\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"8262b103-7c5a-40c8-9681-3282084e7b13\"}" timestamp=1689544991122
Jul 17 03:04:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:24.370Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_restart_policy\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"cm-policy-local-cache-redis-master-69967b5879-8m6nj\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", type=\"Always\", uid=\"ebdf9219-42ed-4a26-8500-14f2dacdb9fe\"}" timestamp=1689544991122
Jul 17 03:04:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:24.411Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"firewall\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", failure_type=\"pgmajfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"hierarchy\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689544993984
Jul 17 03:04:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:24.594Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"tmpfs\", name=\"data-var-lib-kubelet-pods-ebdf9219\\\\x2d42ed\\\\x2d4a26\\\\x2d8500\\\\x2d14f2dacdb9fe-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2ddcn6t.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689544991102
Jul 17 03:04:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:24.756Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dracut-pre-pivot.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"oneshot\", type=\"service\"}" timestamp=1689544991102
Jul 17 03:04:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:24.789Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"pcap-tsgx12\", asset_id=\"117\", condition=\"true\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"local-path-provisioner-957fdf8bc-bc8qf\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"89567228-6ae4-4ed5-8117-7627ccbebb6d\"}" timestamp=1689544991122
Jul 17 03:04:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:24.809Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_info\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_endpoint=\"prometheus-kube-state-metrics\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689544991122
Jul 17 03:04:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:24.836Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx09\", asset_id=\"55\", container=\"dashboard-metrics-scraper\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Sync\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689544992671
Jul 17 03:04:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:25.045Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"nsfs\", name=\"run-netns-cni\\\\x2dfab231b6\\\\x2d5371\\\\x2d5382\\\\x2d1812\\\\x2d98b0c2a68a7d.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689544991102
Jul 17 03:04:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:25.087Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_fifo_total\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"bond0\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544994790
Jul 17 03:04:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:25.128Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"auth-rpcgss-module.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689544998416
Jul 17 03:04:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:25.154Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Running\", pod=\"helm-install-systemd-exporter-zg78t\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"b4187347-4485-4cc4-92e8-53ea7fe72a5f\"}" timestamp=1689544991122
Jul 17 03:04:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:25.189Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"pcap-tsgx12\", asset_id=\"117\", condition=\"false\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-kube-state-metrics-6b768c7bf8-nfws4\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"99e215df-15ea-44bd-824d-42fb75c6e74b\"}" timestamp=1689544991122
Jul 17 03:04:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:25.440Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_commands_duration_seconds_total\", asset=\"msh-tsgx09\", asset_id=\"55\", cmd=\"zrangebyscore\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689544997019
Jul 17 03:04:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:25.490Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_created\", asset=\"pcap-tsgx12\", asset_id=\"117\", configmap=\"kube-root-ca.crt\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"consul\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689544991122
Jul 17 03:04:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:25.683Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db21\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545001795
Jul 17 03:04:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:25.736Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas_ready\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"metrics-server\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689544991122
Jul 17 03:04:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:25.936Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"rshim.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"service\"}" timestamp=1689544991102
Jul 17 03:04:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:25.943Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db188\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689544997019
Jul 17 03:04:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:25.976Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-ens3f0np0.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689544991102
Jul 17 03:04:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:26.014Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Tcp_Close_byK\", app_name=\"sapp4\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", vsysId=\"1\"}" timestamp=1689544998016
Jul 17 03:04:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:26.069Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-9379d048\\\\x2d15af\\\\x2d4f3f\\\\x2dae10\\\\x2d5d1f0ef6e79a-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2dwxzs6.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689544998416
Jul 17 03:04:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:26.318Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_usage_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"dign-server-dns\", cpu=\"total\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545003421
Jul 17 03:04:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:26.515Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db149\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545003549
Jul 17 03:04:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:26.626Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dracut-pre-pivot.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"service\"}" timestamp=1689544998416
Jul 17 03:04:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:26.743Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kdump.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689544991102
Jul 17 03:04:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:26.878Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_working_set_bytes\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"prometheus-node-exporter\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545001588
Jul 17 03:04:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:26.901Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_complete\", asset=\"pcap-tsgx12\", asset_id=\"117\", condition=\"unknown\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", job_name=\"helm-install-mrzcpd-exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689544991122
Jul 17 03:04:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:27.322Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"54\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544995810
Jul 17 03:04:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:27.337Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"85\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544995810
Jul 17 03:04:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:27.663Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-ens8f0np0.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689544994114
Jul 17 03:04:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:27.705Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_cpu_period\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"dign-server-dns\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545004841
Jul 17 03:04:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:27.981Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_user_seconds_total\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"firewall\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689544999257
Jul 17 03:04:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:28.023Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_net_dev_group\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth6d9f498e\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545002882
Jul 17 03:04:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:28.106Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_drop_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"virtio_dign_s\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545002882
Jul 17 03:04:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:28.110Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_errs_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth9db5012c\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545002882
Jul 17 03:04:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:28.470Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_running\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"helm\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-traefik-rdmft\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"3daa5d94-8d7c-4d71-883e-255619668328\"}" timestamp=1689544995682
Jul 17 03:04:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:28.548Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_metadata_resource_version\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"chart-values-tsg-traffic-engine-numa-1\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689544995682
Jul 17 03:04:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:28.563Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"27\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545002882
Jul 17 03:04:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:28.624Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_replicas\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-1-firewall-58d457d665\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689544998744
Jul 17 03:04:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:28.642Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_labels\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"chart-values-systemd-exporter\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689544998744
Jul 17 03:04:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:28.656Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"94\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544995810
Jul 17 03:04:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:28.670Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failcnt\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"dign-server-web\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689544988686
Jul 17 03:04:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:28.686Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"31\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545002882
Jul 17 03:04:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:28.699Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_compressed_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens2f1np1\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545002882
Jul 17 03:04:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:28.741Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_fully_labeled_replicas\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"prometheus-kube-state-metrics-6b768c7bf8\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689544998744
Jul 17 03:04:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:28.782Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_created\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_endpoint=\"prometheus-server\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689544995682
Jul 17 03:04:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:28.786Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_gc_duration_seconds\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", quantile=\"0\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545003549
Jul 17 03:04:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:28.877Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544995810
Jul 17 03:04:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:28.914Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-podb2322992_daa4_43ba_897c_0532360cf5ea.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"slice\"}" timestamp=1689544993907
Jul 17 03:04:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:28.968Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-br_dign_s.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689544993907
Jul 17 03:04:29 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:29.082Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_gc_heap_allocs_by_size_bytes_bucket\", agent_name=\"msh-kvm-nz001\", application=\"loki\", datacenter=\"MSH\", instance=\"127.0.0.1:13100\", le=\"13568.999999999998\"}" timestamp=1689545008327
Jul 17 03:04:29 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:29.161Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_limit_bytes\", asset=\"twa-tsgx09\", asset_id=\"107\", container=\"monit-stream\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545005490
Jul 17 03:04:29 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:29.183Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_info\", asset=\"twa-tsgx09\", asset_id=\"107\", board_id=\"MT_0000000359\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_10\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", firmware_version=\"22.28.4000\", hca_type=\"MT4125\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544995810
Jul 17 03:04:30 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:30.537Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"prometheus-node-exporter\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", failure_type=\"pgmajfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"hierarchy\"}" timestamp=1689544992222
Jul 17 03:04:30 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:30.594Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_errors_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", interface=\"eno34np1\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545003631
Jul 17 03:04:30 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:30.629Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_oom_events_total\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"prometheus-node-exporter\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689544992222
Jul 17 03:04:30 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:30.642Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"50\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545002882
Jul 17 03:04:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:31.015Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_last_seen\", asset=\"msh-tsgx05\", asset_id=\"67\", container=\"exporter-merger\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689544995861
Jul 17 03:04:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:31.172Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"getty@tty1.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689545001015
Jul 17 03:04:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:31.192Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_spec_replicas\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"cm-policy-local-cache-redis-master\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545004068
Jul 17 03:04:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:31.211Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_fifo_total\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens4f2np2\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545000939
Jul 17 03:04:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:31.237Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_carrier_total\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"vethe271b72a\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545000939
Jul 17 03:04:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:31.260Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_load_average_10s\", asset=\"msh-tsgx05\", asset_id=\"67\", container=\"traefik\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689544986585
Jul 17 03:04:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:31.318Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx05\", asset_id=\"67\", container=\"configmap-reloader\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Sync\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689544994621
Jul 17 03:04:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:31.318Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_created\", asset=\"msh-tsgx05\", asset_id=\"67\", configmap=\"chart-content-tsg-traffic-engine-numa-1\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545004068
Jul 17 03:04:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:31.320Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx05\", asset_id=\"67\", container=\"coredns\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Read\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689544991052
Jul 17 03:04:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:31.335Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"getty.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"target\"}" timestamp=1689545001015
Jul 17 03:04:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:31.335Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db18\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545010141
Jul 17 03:04:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:31.344Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_device_id\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens4f3np3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545000939
Jul 17 03:04:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:31.360Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-d8012c00\\\\x2daaec\\\\x2d4f56\\\\x2da35b\\\\x2dad6f4d1ef590-volume\\\\x2dsubpaths-config\\\\x2dvolume-telegraf-1.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689545001015
Jul 17 03:04:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:31.375Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_drop_total\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens2f0np0\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545000939
Jul 17 03:04:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:31.643Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_persistentvolumeclaim_info\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", persistentvolumeclaim=\"prometheus-alertmanager\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", storageclass=\"local-path\", volumename=\"pvc-a3534228-dbb7-4e18-b082-72065af86b7a\"}" timestamp=1689545010436
Jul 17 03:04:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:31.837Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_multicast_packets_transmitted_total\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_7\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544999093
Jul 17 03:04:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:31.846Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_running\", asset=\"twa-tsgx03\", asset_id=\"95\", container=\"telegraf-security\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", pod=\"tsg-traffic-engine-numa-0-firewall-757c9d8dcf-h8b6r\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"6bb2158f-52d5-4138-9127-38341f63d806\"}" timestamp=1689545010436
Jul 17 03:04:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:31.855Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_file_descriptors\", asset=\"msh-tsgx05\", asset_id=\"67\", container=\"prometheus-node-exporter\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689544979963
Jul 17 03:04:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:31.891Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"106\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544999093
Jul 17 03:04:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:31.892Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_dormant\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethf7cbe37a\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544999093
Jul 17 03:04:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:31.927Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", fstype=\"ext4\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/data/var/lib/kubelet/pods/1df6bd2b-0fea-4549-94c2-ddadc3bbc3e4/volume-subpaths/telegraf/telegraf-security/0\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544999093
Jul 17 03:04:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:31.941Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"77\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544999093
Jul 17 03:04:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:31.968Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_files_free\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"shm\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/f590a5837d5ab6427be27f6d4fc128fb5b85feec0e43676a9400323a286ec0ed/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545000791
Jul 17 03:04:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:32.033Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_running\", asset=\"twa-tsgx03\", asset_id=\"95\", container=\"dign-server-dns\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"dign-server-p4cvw\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"b3068a5c-f8e2-4c2d-89be-7648e87ab2c1\"}" timestamp=1689545010436
Jul 17 03:04:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:32.045Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"119\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544999093
Jul 17 03:04:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:32.045Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"var-lib-kubelet-pods-88109228\\\\x2d9032\\\\x2d4124\\\\x2d8904\\\\x2da72a5d84bc7b-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2dxn46j.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689545006907
Jul 17 03:04:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:32.138Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2duuid-b9d80402\\\\x2db888\\\\x2d4dd2\\\\x2dbd97\\\\x2df8d7510dbd36.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689545001015
Jul 17 03:04:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:32.241Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_bytes_total\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethc8d1fba9\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544999093
Jul 17 03:04:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:32.287Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"86\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544999093
Jul 17 03:04:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:32.300Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"overlay\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-33a39fbfdbd335dcdb651bbfbf8b17fd887173b1026bc67aa44394c7f3b3fe67-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689545006907
Jul 17 03:04:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:32.366Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Udp_Offload_STM\", app_name=\"sapp4\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", vsysId=\"1\"}" timestamp=1689545002297
Jul 17 03:04:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:32.433Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"6\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545000791
Jul 17 03:04:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:32.445Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"66\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545000791
Jul 17 03:04:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:32.506Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_ready\", asset=\"msh-tsgx05\", asset_id=\"67\", container=\"prometheus-server\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-server-6ff784d758-rmvjl\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"0fa5c602-236e-4657-a253-bfddabdcc9f1\"}" timestamp=1689545004068
Jul 17 03:04:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:32.596Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"cm-policy-local-cache-redis-master-69967b5879-rj8lj\", project=\"TSG-OS\", project_id=\"7\", reason=\"NodeAffinity\", service=\"prometheus-kube-state-metrics\", uid=\"adf25328-be5b-4a8f-99a7-ca39872df08e\"}" timestamp=1689545004068
Jul 17 03:04:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:32.631Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_reads_bytes_total\", asset=\"twa-tsgx05\", asset_id=\"99\", container=\"dign-server-dns\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689544997122
Jul 17 03:04:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:32.633Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_created\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", uid=\"897f7507-3cfb-43d0-8d42-87a07682f47c\"}" timestamp=1689544999894
Jul 17 03:04:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:32.752Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"110\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544999093
Jul 17 03:04:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:32.770Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_sockstat_FRAG6_memory\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545000605
Jul 17 03:04:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:32.826Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-infiniband-umad7.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689545007691
Jul 17 03:04:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:32.860Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_defrag_key_hits\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545010141
Jul 17 03:04:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:32.946Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db131\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545010141
Jul 17 03:04:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:32.966Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_owner\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", owner_is_controller=\"true\", owner_kind=\"Deployment\", owner_name=\"tsg-traffic-engine-numa-0-firewall\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-0-firewall-7cbb5bf99c\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545010436
Jul 17 03:04:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:32.966Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_annotations\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"chart-values-prometheus\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689544999894
Jul 17 03:04:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:33.015Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-3335393392fe8776cd087fa01c9cf652fdc388d8cc2f05f12e77463bb2972e84.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"scope\"}" timestamp=1689545006907
Jul 17 03:04:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:33.032Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filefd_allocated\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545000605
Jul 17 03:04:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:33.052Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_flags\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens1f0np0\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545000605
Jul 17 03:04:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:33.193Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_errors_total\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"cadvisor\", interface=\"ens4f0np0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689544992525
Jul 17 03:04:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:33.196Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-netns-cni\\\\x2de70f34a3\\\\x2d827a\\\\x2dd7ef\\\\x2dc77b\\\\x2d3ea6cadb82a7.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689545006907
Jul 17 03:04:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:33.267Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_protocol_type\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth28573ba4\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545000605
Jul 17 03:04:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:33.288Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"106\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545000791
Jul 17 03:04:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:33.296Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_info\", asset=\"twa-tsgx05\", asset_id=\"99\", container=\"local-path-provisioner\", container_id=\"containerd://802f51fc4fce8d58c7b11debed5c4bd03ad3dc298cdb288b92579316633a97bf\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", image=\"docker.io/rancher/local-path-provisioner:v0.0.24\", image_id=\"sha256:b29384aeb4b13e047448ccfd312c52b4d023abcbbaafcab174293a97821dddb0\", image_spec=\"rancher/local-path-provisioner:v0.0.24\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"local-path-provisioner-957fdf8bc-rf5f7\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"f2034e38-7d8b-4f82-8274-cb5f14eacda8\"}" timestamp=1689544999894
Jul 17 03:04:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:33.350Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"0\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689544999093
Jul 17 03:04:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:33.370Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"twa-tsgx05\", asset_id=\"99\", condition=\"unknown\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"dign-server-nrgkb\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"48017225-2801-4344-bec5-b6556a71e4cf\"}" timestamp=1689544999894
Jul 17 03:04:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:33.409Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Dup_Key_Err\", app_name=\"sapp4\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", vsysId=\"1\"}" timestamp=1689545003815
Jul 17 03:04:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:33.413Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"41\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545000605
Jul 17 03:04:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:33.442Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-29a02c5616c36efa54d50fd347b786192360e0db5698beb8b3281f8434f97889.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"scope\"}" timestamp=1689545007691
Jul 17 03:04:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:33.474Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_labels\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-0-firewall-687fb44465\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545010436
Jul 17 03:04:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:33.492Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"23\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545000605
Jul 17 03:04:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:33.585Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_frame_total\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"vethf7e6d432\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545000791
Jul 17 03:04:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:33.587Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"var-lib-kubelet-pods-9982afd0\\\\x2d5736\\\\x2d4dce\\\\x2db764\\\\x2d2f7b0c30831a-volume\\\\x2dsubpaths-config\\\\x2dvolume-telegraf-2.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689545007691
Jul 17 03:04:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:33.612Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_packets_total\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"cadvisor\", interface=\"eth0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689544998584
Jul 17 03:04:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:33.630Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_terminated\", asset=\"twa-tsgx03\", asset_id=\"95\", container=\"helm\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-traefik-crd-xvctc\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"037f9a49-f9c9-47d9-b6d3-eeb2a6dc570c\"}" timestamp=1689545010436
Jul 17 03:04:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:33.907Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_info\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"prometheus-server\", container_id=\"containerd://36bc21e307f94721d2ae3da9e6bac494552d446d3fc34a666e9887b93bfa6d28\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", image=\"quay.io/prometheus/prometheus:v2.39.1\", image_id=\"sha256:6b9895947e9e4b2db6b271f999b3637e660185ae331021c3d47e6e6345a1c472\", image_spec=\"quay.io/prometheus/prometheus:v2.39.1\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-server-6ff784d758-mv9fn\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"461d1588-9ea2-41cf-8eca-daa78ce6d555\"}" timestamp=1689544999276
Jul 17 03:04:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:33.931Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"58\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545000791
Jul 17 03:04:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:34.179Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_info\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"chart-values-mrzcpd-exporter\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689544999894
Jul 17 03:04:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:34.585Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_usage_bytes\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/shm\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689544989057
Jul 17 03:04:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:34.595Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_writes_bytes_total\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689544998082
Jul 17 03:04:38 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:38.259Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"jvm_memory_pool_bytes_committed\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-Hmaster\", endpoint_id=\"341\", module=\"DC-HBase\", module_id=\"3\", nz_agent_id=\"3\", pool=\"CMS Old Gen\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545016971
Jul 17 03:04:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:40.475Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"scrape_series_added\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_instance=\"10.42.0.61:9005\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"mrzcpd-exporter\"}" timestamp=1689545008913
Jul 17 03:04:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:41.061Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_created\", asset=\"twa-tsgx04\", asset_id=\"97\", configmap=\"sce-tsg-traffic-engine-numa-0\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545020034
Jul 17 03:04:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:41.099Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"43\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545011647
Jul 17 03:04:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:41.152Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_10_194_1_metric_incrementTime_90th_percentile\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545019759
Jul 17 03:04:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:41.217Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_last_seen\", asset=\"twa-tsgx04\", asset_id=\"97\", container=\"lb-tcp-443\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545008816
Jul 17 03:04:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:41.239Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_usage_bytes\", asset=\"twa-tsgx04\", asset_id=\"97\", container=\"telegraf\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689544995078
Jul 17 03:04:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:41.278Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"90\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545011647
Jul 17 03:04:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:41.295Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"16\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545011647
Jul 17 03:04:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:41.485Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_10_194_1_metric_putTime_mean\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545019759
Jul 17 03:04:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:41.518Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_packets_transmitted_total\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_1\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545011647
Jul 17 03:04:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:41.521Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_TASK_metric_incrementTime_75th_percentile\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545019759
Jul 17 03:04:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:41.538Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_flags\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth4e7244c8\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545011647
Jul 17 03:04:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:41.571Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_compressed_total\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens8f1np1\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545011647
Jul 17 03:04:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:41.686Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_memStoreSize\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"firewall_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"6b0ca914ac067acc26419d3a7e6877ce\", table=\"index_filename_firewall_hos_bucket\", type=\"index\"}" timestamp=1689545019759
Jul 17 03:04:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:41.711Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_read_seconds_total\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda3\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545004441
Jul 17 03:04:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:41.730Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_cpu_period\", asset=\"twa-tsgx04\", asset_id=\"97\", container=\"monit-stream\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545008797
Jul 17 03:04:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:41.732Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_inodes_total\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/sys/fs/cgroup\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545004441
Jul 17 03:04:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:41.908Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"twa-tsgx04\", asset_id=\"97\", condition=\"false\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-pushgateway-bd476bcf9-j8zz6\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"7c06d85f-872f-4519-ad0f-41e822d881d4\"}" timestamp=1689545020034
Jul 17 03:04:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:42.472Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2did-scsi\\\\x2dSDELL_PERC_H345_Adp_00eaf83a825e7c2c2c00558284f0a7ce\\\\x2dpart5.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689545020671
Jul 17 03:04:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:42.709Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_flushesQueuedCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"c797c1b183075738b860976f05c9f684\", table=\"knowledge_base_hos_bucket\", type=\"data\"}" timestamp=1689545019759
Jul 17 03:04:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:42.818Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-rdma-devices-mlx5_4:1-umad.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689545020671
Jul 17 03:04:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:42.929Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Pkt_Outbound\", app_name=\"sapp4\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", vsysId=\"1\"}" timestamp=1689545007581
Jul 17 03:04:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:43.069Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"rpc-statd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689545020671
Jul 17 03:04:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:43.114Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-pod1d9c1047_30bc_4de8_bd00_07e7917cd6b7.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"slice\"}" timestamp=1689545020671
Jul 17 03:04:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:43.206Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-sda.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689545020671
Jul 17 03:04:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:43.220Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-sda5.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689545020671
Jul 17 03:04:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:43.287Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-pod4d40e67b_cb75_462e_b6c7_c667d37b9d55.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"slice\"}" timestamp=1689545020671
Jul 17 03:04:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:43.477Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"nfs-utils.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"service\"}" timestamp=1689545020671
Jul 17 03:04:44 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:44.094Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_tasks_current\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-remount-fs.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689545020671
Jul 17 03:04:44 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:44.177Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", cpu=\"72\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545016242
Jul 17 03:04:44 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:44.202Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", cpu=\"75\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545016242
Jul 17 03:04:44 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:44.280Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_packets_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"cadvisor\", interface=\"ens4f2np2\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545018768
Jul 17 03:04:44 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:44.301Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_net_dev_group\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens1f1np1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545016242
Jul 17 03:04:44 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:44.306Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_drop_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth9d1dd258\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545016242
Jul 17 03:04:44 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:44.342Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_fifo_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth8d5344fb\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545016242
Jul 17 03:04:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:49.370Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_2_metric_putTime_98th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545027438
Jul 17 03:04:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:49.815Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_created\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", uid=\"3ce80ecf-5a44-4ded-bef6-4d14d2334a70\"}" timestamp=1689545021568
Jul 17 03:04:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:49.835Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_scaling_frequency_min_hertz\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"114\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545026709
Jul 17 03:04:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:49.949Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_194_1_metric_putTime_mean\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545027438
Jul 17 03:04:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:49.965Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_3_metric_appendTime_99th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545027438
Jul 17 03:04:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:50.064Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-initctl.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689545026554
Jul 17 03:04:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:50.080Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_io_current\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"overlay_0-471\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545023504
Jul 17 03:04:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:50.090Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"kube-state-metrics\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Async\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545012468
Jul 17 03:04:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:50.097Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_scaling_frequency_min_hertz\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"15\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545026709
Jul 17 03:04:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:50.154Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_reads_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"overlay_0-398\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545023504
Jul 17 03:04:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:50.155Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_writes_total\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/run\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545015240
Jul 17 03:04:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:50.172Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"prometheus-server-configmap-reload\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", failure_type=\"pgfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"container\"}" timestamp=1689545006891
Jul 17 03:04:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:50.191Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"mcelog.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"simple\", type=\"service\"}" timestamp=1689545026554
Jul 17 03:04:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:50.197Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"cadvisor_version_info\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", kernelVersion=\"5.17.15-1.el8.x86_64\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", osVersion=\"TSG-OS v23.07-rc1-079e964 (TSGXNXR620G40R01P0906)\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545019050
Jul 17 03:04:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:50.205Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_writes_merged_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/96a9f56f8880670d84ba1753f731c63c62abdde31dae3ba4a5cb169300591055/shm\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545023504
Jul 17 03:04:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:50.230Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Running\", pod=\"helm-install-ipmi-exporter-pspfb\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"8d60fd58-fc4a-4fec-a27c-e595a24480b5\"}" timestamp=1689545021568
Jul 17 03:04:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:50.242Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"twa-tsgx10\", asset_id=\"109\", condition=\"unknown\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"systemd-exporter-7f549d9485-qjwvl\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"d1333293-1dd6-450f-8c65-92ef3d57012b\"}" timestamp=1689545019668
Jul 17 03:04:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:50.251Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-207ceb95\\\\x2de1ef\\\\x2d4b7e\\\\x2da587\\\\x2d0a6495b0f134-volume\\\\x2dsubpaths-telegraf-telegraf\\\\x2dsecurity-0.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689545026554
Jul 17 03:04:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:50.343Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"twa-tsgx10\", asset_id=\"109\", cpu=\"65\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545021797
Jul 17 03:04:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:50.362Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"rescue.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"target\"}" timestamp=1689545021057
Jul 17 03:04:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:50.434Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_oom_events_total\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"kubernetes-dashboard\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545010102
Jul 17 03:04:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:50.460Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", phase=\"Running\", pod=\"prometheus-kube-state-metrics-6b768c7bf8-rkth4\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"1f4cb14c-70a0-45e7-9cd4-e57b3292287d\"}" timestamp=1689545021568
Jul 17 03:04:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:50.476Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_type\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"kubernetes-dashboard-certs\", service=\"prometheus-kube-state-metrics\", type=\"Opaque\"}" timestamp=1689545021568
Jul 17 03:04:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:50.507Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:40-0000:40:01.1-0000:41:00.0-net-ens3f0np0.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689545026554
Jul 17 03:04:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:50.669Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"tmpfs\", name=\"run-k3s-containerd-io.containerd.grpc.v1.cri-sandboxes-35aa6bf90a7480a32af757eb89152be3c5abda69125eb447238a0e300f41a001-shm.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689545021057
Jul 17 03:04:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:50.685Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"21\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545026709
Jul 17 03:04:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:50.688Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"mlocate-updatedb.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"service\"}" timestamp=1689545021057
Jul 17 03:04:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:50.730Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"40\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545026709
Jul 17 03:04:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:50.816Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx08\", asset_id=\"105\", container=\"configmap-reloader\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Async\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545020927
Jul 17 03:04:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:50.830Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"twa-tsgx08\", asset_id=\"105\", container=\"prometheus-pushgateway\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"sleeping\"}" timestamp=1689545024209
Jul 17 03:04:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:50.871Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_namespace_created\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-node-lease\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545021568
Jul 17 03:04:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:50.888Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_free_bytes\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"shm\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/f9018ed6bd85ccf87fa84a57ebe1e8f139adfc841aed3294bca3ee381fea5005/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545026709
Jul 17 03:04:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:50.895Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"twa-tsgx10\", asset_id=\"109\", condition=\"true\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"coredns-766944d45b-dstvr\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"b52de515-fecd-439f-b56f-e8953f782519\"}" timestamp=1689545019668
Jul 17 03:04:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:50.906Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_writes_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/data/var/lib/kubelet/pods/fe1b9683-356b-448d-8311-9cb2e5a90800/volumes/kubernetes.io~projected/kube-api-access-rskgg\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545023504
Jul 17 03:04:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:50.911Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_labels\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.tsg-traffic-engine-numa-0.v2\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545019668
Jul 17 03:04:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:50.941Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_readonly\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"shm\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/0500205352582fe55f0c9607f3e3848e1f23c3fd83667094351e6d099fc71fcc/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545026709
Jul 17 03:04:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:50.959Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_id\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens2f0np0\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545026709
Jul 17 03:04:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:50.991Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_info\", asset=\"twa-tsgx10\", asset_id=\"109\", configmap=\"chart-content-tsg-traffic-engine-numa-0\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545019668
Jul 17 03:04:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:50.993Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545026709
Jul 17 03:04:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:51.004Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_changes_total\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth1dedddbd\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545021797
Jul 17 03:04:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:51.009Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_mtu_bytes\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens7f1np1\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545026709
Jul 17 03:04:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:51.020Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_link_downed_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_2\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545026709
Jul 17 03:04:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:51.028Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_drop_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"lo\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545026709
Jul 17 03:04:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:51.037Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_queue_length\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth9afe06eb\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545026709
Jul 17 03:04:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:51.055Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_speed_bytes\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth66a8f63b\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545026709
Jul 17 03:04:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:51.088Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2did-scsi\\\\x2dSDELL_PERC_H345_Adp_009195b3b23903272c00c14683f0a7ce\\\\x2dpart1.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689545026554
Jul 17 03:04:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:51.106Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-ens4f1np1.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689545026554
Jul 17 03:04:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:51.120Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"abrtd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"service\"}" timestamp=1689545021057
Jul 17 03:04:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:51.136Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"tsg_master_del\", app_name=\"tsg_master\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"traffic-engine\", line_name=\"subscriber\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", table_name=\"sync_exdata\", vsysId=\"1\"}" timestamp=1689545018548
Jul 17 03:04:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:51.236Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_bytes_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", interface=\"eth0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689545013755
Jul 17 03:04:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:51.248Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_reads_merged_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"overlay_0-398\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545023504
Jul 17 03:04:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:51.263Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_terminated\", asset=\"twa-tsgx08\", asset_id=\"105\", container=\"kubernetes-dashboard\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", pod=\"kubernetes-dashboard-66dd5b456-5x4qv\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"a0e438ae-62dd-4767-974d-b0821c7c31f3\"}" timestamp=1689545021568
Jul 17 03:04:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:51.437Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"25\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545026709
Jul 17 03:04:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:51.452Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_mtu_bytes\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth1e5d3348\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545026709
Jul 17 03:04:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:53.069Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db183\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545030108
Jul 17 03:04:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:53.266Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx08\", asset_id=\"65\", cpu=\"56\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545028328
Jul 17 03:04:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:53.272Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", phase=\"Running\", pod=\"prometheus-pushgateway-bd476bcf9-8t7vj\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"f2baf82c-b91b-46ec-8a59-8f55f1b86285\"}" timestamp=1689545024478
Jul 17 03:04:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:53.316Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"msh-tsgx08\", asset_id=\"65\", cpu=\"57\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545028328
Jul 17 03:04:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:53.320Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled_time\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"configmap-reloader-6c55bbc49f-qwg7z\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"a0009b6f-7978-40ef-a946-c18d7bd61697\"}" timestamp=1689545024478
Jul 17 03:04:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:53.437Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_waiting\", asset=\"pcap-tsgx11\", asset_id=\"115\", container=\"helm\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-cm-policy-local-cache-xlkqt\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"4f506bcb-6f14-4c9c-9398-6b207614efb5\"}" timestamp=1689545024478
Jul 17 03:04:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:53.453Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth2dc20da0\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545029337
Jul 17 03:04:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:53.469Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_address_assign_type\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth671204b7\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545028328
Jul 17 03:04:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:53.492Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_changes_total\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"eno34np1\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545028328
Jul 17 03:04:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:53.536Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx08\", asset_id=\"65\", cpu=\"61\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545028328
Jul 17 03:04:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:53.537Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_system_seconds_total\", asset=\"msh-tsgx08\", asset_id=\"65\", container=\"dign-client\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545005991
Jul 17 03:04:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:53.552Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_device_id\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"virtio_dign_s\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545028328
Jul 17 03:04:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:53.556Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"118\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545023391
Jul 17 03:04:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:53.562Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_oom_events_total\", asset=\"msh-tsgx12\", asset_id=\"113\", container=\"kubernetes-dashboard\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545013777
Jul 17 03:04:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:53.604Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_ulimits_soft\", asset=\"msh-tsgx12\", asset_id=\"113\", container=\"mrapm-device\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", ulimit=\"max_open_files\"}" timestamp=1689545012694
Jul 17 03:04:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:53.610Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx08\", asset_id=\"65\", cpu=\"96\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545028328
Jul 17 03:04:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:53.622Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_threads\", asset=\"msh-tsgx08\", asset_id=\"65\", container=\"traefik\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545012241
Jul 17 03:04:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:53.633Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_edac_correctable_errors_total\", asset=\"pcap-dellr740-dt002\", asset_id=\"33\", controller=\"1\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"NC-Gohangout-pcap-dellr740-dt002\", endpoint_id=\"173\", module=\"DC-Gohangout\", module_id=\"35\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545032667
Jul 17 03:04:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:53.643Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_waiting\", asset=\"msh-tsgx12\", asset_id=\"113\", container=\"lb-tcp-443\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"svclb-traefik-ad22eefe-x9glh\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"7c7040d2-bb49-4f76-ac55-4ff20b58ef7e\"}" timestamp=1689545028663
Jul 17 03:04:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:53.693Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_up\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth3449d361\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545028328
Jul 17 03:04:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:53.714Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_resident_memory_bytes\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"chronyd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689545032135
Jul 17 03:04:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:53.718Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_device_id\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens8f1np1\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545028328
Jul 17 03:04:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:53.722Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Running\", pod=\"svclb-traefik-ab31cfb4-hhks6\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"44ba41ae-6294-4471-a8bc-5c98a2a4eb5d\"}" timestamp=1689545028950
Jul 17 03:04:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:53.748Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_netstat_Udp6_InErrors\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545019409
Jul 17 03:04:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:53.765Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_labels\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"470c0f70-ccd5-440d-a141-0a77d5cf876f\"}" timestamp=1689545028950
Jul 17 03:04:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:53.798Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_receive_remote_physical_errors_total\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_5\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545023391
Jul 17 03:04:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:53.805Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_read_seconds_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda3\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545029663
Jul 17 03:04:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:53.814Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Eth_Bit\", app_name=\"sapp4\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", vsysId=\"1\"}" timestamp=1689545022106
Jul 17 03:04:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:53.842Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-udev-settle.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689545032135
Jul 17 03:04:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:53.853Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"metrics-server\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545028950
Jul 17 03:04:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:53.887Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_labels\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"kubernetes-dashboard-csrf\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545028663
Jul 17 03:04:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:53.902Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx12\", asset_id=\"113\", cpu=\"73\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545023391
Jul 17 03:04:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:53.934Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_MetaHlogSplitTime_90th_percentile\", asset=\"msh-dellr740-dt003\", asset_id=\"9\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt003-Hmaster\", endpoint_id=\"385\", module=\"DC-HBase\", module_id=\"3\", name=\"Master\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"FileSystem\"}" timestamp=1689545032402
Jul 17 03:04:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:53.965Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_speed_bytes\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth149fb130\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545023391
Jul 17 03:04:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:53.978Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_up\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"eno1\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545023391
Jul 17 03:04:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:53.990Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_timex_pps_frequency_hertz\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545023391
Jul 17 03:04:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:53.992Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", cpu=\"108\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545029337
Jul 17 03:04:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:53.997Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_net_dev_group\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens7f1np1\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545023391
Jul 17 03:04:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:54.016Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_resident_memory_bytes\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"gpm.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689545032135
Jul 17 03:04:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:54.018Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"iscsi.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"oneshot\", type=\"service\"}" timestamp=1689545020884
Jul 17 03:04:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:54.043Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx08\", asset_id=\"65\", cpu=\"62\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545028328
Jul 17 03:04:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:54.057Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_metadata_generation\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"prometheus-alertmanager\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545028663
Jul 17 03:04:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:54.081Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx08\", asset_id=\"65\", cpu=\"108\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545028328
Jul 17 03:04:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:54.112Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_status_failed\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", job_name=\"helm-install-tsg-traffic-engine-numa-0\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545022058
Jul 17 03:04:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:54.143Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_netstat_Udp_InErrors\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545029337
Jul 17 03:04:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:54.164Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db91\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545018659
Jul 17 03:04:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:54.172Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2dpartuuid-68ac4f25\\\\x2d0269\\\\x2d4511\\\\x2dac03\\\\x2db8bc40ab4e28.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689545032135
Jul 17 03:04:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:54.276Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_threads\", asset=\"msh-tsgx12\", asset_id=\"113\", container=\"firewall\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689545014735
Jul 17 03:04:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:54.294Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_hwmon_temp_celsius\", asset=\"msh-tsgx12\", asset_id=\"113\", chip=\"pci0000:00_0000:00:19_3\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", sensor=\"temp10\", service=\"prometheus-node-exporter\"}" timestamp=1689545023391
Jul 17 03:04:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:54.316Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"overlay\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-5b2e5aa8455ace000f11eec393f2841e58cba3391f46a8ba95d2a64f4eb7a648-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689545020884
Jul 17 03:04:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:54.318Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.grpc.v1.cri-sandboxes-f2304a5f013e04fcbb7f4b33857322eb4e38a481d97c494dcd44080e215859d0-shm.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689545025496
Jul 17 03:04:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:54.341Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_mapped_file\", asset=\"pcap-tsgx11\", asset_id=\"115\", container=\"dign-server-web\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545024391
Jul 17 03:04:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:54.400Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_carrier_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth75ed90fc\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545029337
Jul 17 03:04:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:54.423Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db172\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545019367
Jul 17 03:04:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:54.434Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_dormant\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens1f0np0\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545028328
Jul 17 03:04:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:54.469Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_namespace_status_phase\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", phase=\"Active\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545028663
Jul 17 03:04:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:54.484Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-delete-tsg-traffic-engine-numa-0-7gg5s\", project=\"TSG-OS\", project_id=\"7\", reason=\"NodeAffinity\", service=\"prometheus-kube-state-metrics\", uid=\"6d5094f5-cbce-4a02-92f8-9c472f1ba305\"}" timestamp=1689545028663
Jul 17 03:04:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:54.507Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_waiting\", asset=\"msh-tsgx12\", asset_id=\"113\", container=\"coredns\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"coredns-6cb546fb97-4zrr7\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"5782b922-edbe-45a2-a171-c3d8e1c63480\"}" timestamp=1689545028663
Jul 17 03:04:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:54.524Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Udp_Bypass_Pkt\", app_name=\"sapp4\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", vsysId=\"1\"}" timestamp=1689545024735
Jul 17 03:04:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:54.562Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"msh-tsgx08\", asset_id=\"65\", container=\"telegraf\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", state=\"running\"}" timestamp=1689545003205
Jul 17 03:04:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:54.575Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_lookups_total\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545028328
Jul 17 03:04:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:54.623Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_labels\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"cm-policy-local-cache-redis-master\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545028950
Jul 17 03:04:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:54.664Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_carrier_total\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"vethab8afd71\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545028328
Jul 17 03:04:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:54.691Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2did-wwn\\\\x2d0x62cea7f0add59d002c2c317c692f9f02\\\\x2dpart3.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689545031179
Jul 17 03:04:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:54.888Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_info\", asset=\"pcap-tsgx07\", asset_id=\"85\", container=\"helm\", container_id=\"containerd://80d51def75ff8ca3b57bf8f64e00b099605b9e8a38e9e14da0181d08dfb1b2b0\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", image=\"docker.io/rancher/klipper-helm:v0.8.0-build20230510\", image_id=\"sha256:6f42df210d7faa0b6eb56f5f28bb061930c7834af1e7dd4b3586108ec6b1485b\", image_spec=\"rancher/klipper-helm:v0.8.0-build20230510\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-mrzcpd-exporter-k42ln\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"bd8bd8a8-77b3-49ca-b706-0594b012c60c\"}" timestamp=1689545022058
Jul 17 03:04:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:54.967Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_NodeManager_StartContainersNumOps\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-Yarn-msh-dellr740-dt001-NodeManager\", endpoint_id=\"351\", module=\"DC-Yarn\", module_id=\"1\", name=\"RpcDetailedActivityForPort9923\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545033514
Jul 17 03:04:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:54.975Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_data_transmitted_bytes_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"mlx5_1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545019409
Jul 17 03:04:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:55.002Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cpu=\"32\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545019409
Jul 17 03:04:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:55.013Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_commands_failed_calls_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cmd=\"del\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545025184
Jul 17 03:04:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:55.026Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-9f3eaf926bf5a4d1a08231be93ed39891cbd4343fe5db6baf9d3eef54f3aa813.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"scope\"}" timestamp=1689545020884
Jul 17 03:04:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:55.046Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"var-lib-kubelet-pods-8ba148c1\\\\x2df25f\\\\x2d4fa4\\\\x2d82b3\\\\x2d03b9d5a8ace6-volume\\\\x2dsubpaths-config\\\\x2dvolume-telegraf\\\\x2dsecurity-2.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689545031179
Jul 17 03:04:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:55.051Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-virtual-net-virtio_dign_c.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689545031179
Jul 17 03:04:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:55.121Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cpu=\"74\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545019409
Jul 17 03:04:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:55.383Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_swap\", asset=\"pcap-tsgx07\", asset_id=\"85\", container=\"lb-tcp-80\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545030210
Jul 17 03:04:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:55.407Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_usage_bytes\", asset=\"pcap-tsgx07\", asset_id=\"85\", container=\"telegraf\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689545018157
Jul 17 03:04:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:55.732Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"remote-fs-pre.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"target\"}" timestamp=1689545020884
Jul 17 03:04:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:56.280Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"120\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545025883
Jul 17 03:04:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:56.341Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_owner\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", job_name=\"helm-install-cm-policy-local-cache\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", owner_is_controller=\"<none>\", owner_kind=\"<none>\", owner_name=\"<none>\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545033389
Jul 17 03:04:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:56.466Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", container=\"configmap-reloader\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"stopped\"}" timestamp=1689545021938
Jul 17 03:04:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:56.493Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_labels\", asset=\"pcap-tsgx03\", asset_id=\"73\", configmap=\"multus-cni-config\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545033389
Jul 17 03:04:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:56.507Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2dpartuuid-31f63bc7\\\\x2d4d69\\\\x2d4b66\\\\x2d9e97\\\\x2d41d6cc8eb76b.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689545023372
Jul 17 03:04:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:56.507Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sshd-keygen@ecdsa.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689545023372
Jul 17 03:04:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:56.570Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_inodes_free\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda2\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545031471
Jul 17 03:04:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:56.722Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", container=\"dign-server-dns\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Async\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545028978
Jul 17 03:04:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:56.747Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_persistentvolumeclaim_status_phase\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", persistentvolumeclaim=\"prometheus-alertmanager\", phase=\"Bound\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545033389
Jul 17 03:04:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:56.768Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"84\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545025883
Jul 17 03:04:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:57.056Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"66\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545025883
Jul 17 03:04:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:57.164Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"91\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545025883
Jul 17 03:04:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:57.361Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_metadata_resource_version\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.tsg-traffic-engine-numa-0.v3\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545033389
Jul 17 03:04:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:57.472Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_cpu_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-burstable.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"slice\"}" timestamp=1689545023372
Jul 17 03:04:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:57.590Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_errs_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens8f1np1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545025883
Jul 17 03:04:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:57.681Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"pcap-tsgx03\", asset_id=\"73\", condition=\"unknown\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-traefik-crd-blxb5\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"fa6987b7-5e6f-40c1-a86c-e5d47b725265\"}" timestamp=1689545033389
Jul 17 03:04:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:57.763Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_info\", asset=\"pcap-tsgx03\", asset_id=\"73\", cluster_ip=\"10.43.47.157\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", uid=\"58e56b20-d3ad-48f5-9e85-e43263b7ee76\"}" timestamp=1689545033389
Jul 17 03:04:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:57.809Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_compressed_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"br_dign_c\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545025883
Jul 17 03:04:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:57.848Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_user_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", container=\"metrics-server\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545021647
Jul 17 03:04:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:57.860Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_spec_completions\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", job_name=\"helm-install-systemd-exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545033389
Jul 17 03:04:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:57.889Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kmod-static-nodes.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"service\"}" timestamp=1689545023372
Jul 17 03:04:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:57.977Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_mapped_file\", asset=\"pcap-tsgx03\", asset_id=\"73\", container=\"telegraf\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689545016399
Jul 17 03:04:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:59.916Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_cpu_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"snmptrapd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689545024879
Jul 17 03:04:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:04:59.928Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cryptsetup-pre.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"target\"}" timestamp=1689545024879
Jul 17 03:05:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:00.265Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"phy_rx_missed_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", dev=\"ens7f0np0\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"mrzcpd-exporter\"}" timestamp=1689545030377
Jul 17 03:05:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:00.283Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"ext4\", name=\"var-lib-kubelet-pods-93d50d7c\\\\x2dc29d\\\\x2d460a\\\\x2d840d\\\\x2d47c14d4c19f2-volume\\\\x2dsubpaths-config\\\\x2dpath-dign\\\\x2dclient-3.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689545024879
Jul 17 03:05:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:00.291Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"shm\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/4fed3108aa42b99cc67026a1e961df0207e336796f5fe373c0374ff1af88b33c/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545029083
Jul 17 03:05:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:00.303Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-vconsole-setup.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689545024879
Jul 17 03:05:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:00.368Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_metadata_generation\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"traefik\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545027425
Jul 17 03:05:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:00.385Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"91\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545029083
Jul 17 03:05:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:00.530Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-424767a76d4b7e2e71bc8d7381286464bfbeae86b0bf69524fef222fb9e3adee-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689545024879
Jul 17 03:05:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:00.541Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"119\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545029083
Jul 17 03:05:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:00.911Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"gssproxy.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"service\"}" timestamp=1689545024879
Jul 17 03:05:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:00.944Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_reads_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689545033100
Jul 17 03:05:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:00.975Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"99\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545029083
Jul 17 03:05:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:00.976Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"initrd-udevadm-cleanup-db.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"service\"}" timestamp=1689545024879
Jul 17 03:05:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:00.978Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", phase=\"Running\", pod=\"mrzcpd-exporter-5cf7f967cc-qcxqj\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"cfc14401-c454-4aca-ae14-7e249e5a7dc0\"}" timestamp=1689545027425
Jul 17 03:05:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:00.998Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db239\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545025071
Jul 17 03:05:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:00.998Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_daemonset_labels\", asset=\"msh-tsgx01\", asset_id=\"51\", daemonset=\"dign-server\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545027425
Jul 17 03:05:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:00.998Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_aof_current_rewrite_duration_sec\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545025071
Jul 17 03:05:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:01.025Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_max_fds\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"gpm.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689545024879
Jul 17 03:05:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:01.026Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_complete\", asset=\"msh-tsgx01\", asset_id=\"51\", condition=\"false\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", job_name=\"helm-delete-tsg-traffic-engine-numa-0\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545027425
Jul 17 03:05:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:01.042Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2dpartuuid-d74d846e\\\\x2dfe82\\\\x2d4f66\\\\x2da800\\\\x2d2b83dec9dc32.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689545024879
Jul 17 03:05:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:01.150Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"msh-tsgx01\", asset_id=\"51\", condition=\"unknown\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-traefik-pk7nl\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"90133098-1b52-4329-973e-ffb63ab8e8b1\"}" timestamp=1689545027425
Jul 17 03:05:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:01.165Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Tcp_Offload_STM\", app_name=\"sapp4\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", vsysId=\"1\"}" timestamp=1689545028260
Jul 17 03:05:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:01.325Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"103\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545029083
Jul 17 03:05:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:02.417Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_timex_pps_jitter_seconds\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545038318
Jul 17 03:05:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:02.611Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"init.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"scope\"}" timestamp=1689545028447
Jul 17 03:05:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:02.896Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx02\", asset_id=\"63\", cpu=\"103\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545038318
Jul 17 03:05:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:02.977Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"msh-tsgx02\", asset_id=\"63\", cpu=\"96\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545038318
Jul 17 03:05:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:03.268Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"msh-tsgx02\", asset_id=\"63\", condition=\"false\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-systemd-exporter-gn8qh\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"3ac33a80-602d-4e22-a3ab-07460cbdb515\"}" timestamp=1689545031412
Jul 17 03:05:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:03.450Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_info\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"chart-values-prometheus\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545031412
Jul 17 03:05:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:03.458Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"prometheus_target_scrape_pool_targets\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", instance=\"10.10.20.159:10090\", scrape_job=\"asset_ping_7\", type=\"global\"}" timestamp=1689545042990
Jul 17 03:05:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:03.538Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_readonly\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"shm\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/eae093ead7efd40cc68023894f5e82bb1842e37f6341a5751d947c53f08358eb/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545038318
Jul 17 03:05:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:03.554Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_scaling_frequency_max_hertz\", asset=\"msh-dellr740-olap005\", asset_id=\"19\", cpu=\"36\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Spark-msh-dellr740-olap005\", endpoint_id=\"17\", module=\"NC-Spark\", module_id=\"47\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545043391
Jul 17 03:05:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:03.737Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_scrape_collector_duration_seconds\", asset=\"msh-tsgx02\", asset_id=\"63\", collector=\"bcache\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545038318
Jul 17 03:05:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:03.923Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", cpu=\"9\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-dt001\", endpoint_id=\"225\", mode=\"steal\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545042987
Jul 17 03:05:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:03.940Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sdb1\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-dt001\", endpoint_id=\"225\", fstype=\"xfs\", module=\"OLAP-Node-Exporter\", module_id=\"51\", mountpoint=\"/home\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545042987
Jul 17 03:05:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:03.952Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"net_conntrack_dialer_conn_failed_total\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", dialer_name=\"19\", instance=\"10.10.20.159:10090\", reason=\"resolution\", type=\"global\"}" timestamp=1689545042990
Jul 17 03:05:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:03.979Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"prometheus_sd_kubernetes_events_total\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", event=\"delete\", instance=\"10.10.20.159:10090\", role=\"endpoints\", type=\"global\"}" timestamp=1689545042990
Jul 17 03:05:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:04.004Z caller=dedupe.go:112 component=remote level=info remote_name=52db6f url=http://10.10.10.159:10090/nz-agent/prometheus/proxy/api/v1/write msg="Dropped sample for series that was not explicitly dropped via relabelling" ref=4436650
Jul 17 03:05:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:04.179Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-pod11efef4a_05e0_44ee_8681_0e306eccb51c.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"slice\"}" timestamp=1689545028447
Jul 17 03:05:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:04.429Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"prometheus_target_sync_length_seconds\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", instance=\"10.10.20.159:10090\", quantile=\"0.05\", scrape_job=\"asset_ping_87\", type=\"global\"}" timestamp=1689545042990
Jul 17 03:05:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:06.231Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_context_switches_total\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Spark-msh-dellr740-olap003\", endpoint_id=\"13\", module=\"NC-Spark\", module_id=\"47\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545046092
Jul 17 03:05:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:06.419Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_scaling_frequency_max_hertz\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", cpu=\"40\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Spark-msh-dellr740-olap003\", endpoint_id=\"13\", module=\"NC-Spark\", module_id=\"47\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545046092
Jul 17 03:05:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:06.677Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"process_virtual_memory_bytes\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545038290
Jul 17 03:05:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:06.693Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"tsg_master_add\", app_name=\"tsg_master\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"traffic-engine\", line_name=\"compile\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", table_name=\"sync_exdata\", vsysId=\"1\"}" timestamp=1689545044123
Jul 17 03:05:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:06.820Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_colls_total\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens4f2np2\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545043910
Jul 17 03:05:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:06.834Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_up\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth07e82765\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545043910
Jul 17 03:05:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:06.853Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:c0-0000:c0:01.1-0000:c1:00.0-infiniband-mlx5_8.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689545033582
Jul 17 03:05:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:07.023Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_spec_strategy_rollingupdate_max_surge\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"ipmi-exporter\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545041227
Jul 17 03:05:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:07.044Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"cadvisor\", failure_type=\"pgmajfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"container\"}" timestamp=1689545042702
Jul 17 03:05:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:07.062Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2did-scsi\\\\x2d362cea7f085ed36002c25b4d2a14582f7\\\\x2dpart1.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689545033582
Jul 17 03:05:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:07.074Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-netns-cni\\\\x2d3a924f46\\\\x2dcc1e\\\\x2d28b3\\\\x2d919e\\\\x2de670c9925b2b.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689545033582
Jul 17 03:05:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:07.091Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_info\", asset=\"twa-tsgx01\", asset_id=\"91\", configmap=\"telegraf-tsg-traffic-engine-numa-1\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545041227
Jul 17 03:05:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:07.109Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_frequency_min_hertz\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", cpu=\"30\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Spark-msh-dellr740-olap003\", endpoint_id=\"13\", module=\"NC-Spark\", module_id=\"47\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545046092
Jul 17 03:05:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:07.112Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_created\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"ipmi-exporter\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545041227
Jul 17 03:05:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:07.122Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cpu=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545043910
Jul 17 03:05:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:07.129Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"25\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545043910
Jul 17 03:05:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:07.241Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_size_bytes\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"shm\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/a4f0e97497b459a22332f36472a7953a388e47383478f0c8dca1076d2b410fe5/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545043910
Jul 17 03:05:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:07.297Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_address_assign_type\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"bond0\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545043910
Jul 17 03:05:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:07.315Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_name_assign_type\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethb0e8bdf7\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545043910
Jul 17 03:05:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:07.329Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db118\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545038290
Jul 17 03:05:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:07.345Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-netns-cni\\\\x2d2a2d6c94\\\\x2d60c2\\\\x2d23db\\\\x2d4ddb\\\\x2d5b6355c3ac29.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689545033582
Jul 17 03:05:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:07.345Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_max_usage_bytes\", asset=\"twa-tsgx01\", asset_id=\"91\", container=\"monit-stream\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545042945
Jul 17 03:05:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:07.361Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_annotations\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"ipmi-exporter\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545041227
Jul 17 03:05:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:07.410Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"udisks2.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"service\"}" timestamp=1689545033582
Jul 17 03:05:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:07.434Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-server-6ff784d758-76b74\", project=\"TSG-OS\", project_id=\"7\", reason=\"NodeLost\", service=\"prometheus-kube-state-metrics\", uid=\"5ee8d190-8de7-4d53-a103-d0969342cef5\"}" timestamp=1689545041227
Jul 17 03:05:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:07.501Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cpu=\"121\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545043910
Jul 17 03:05:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:07.541Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_free_bytes\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"tmpfs\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545043910
Jul 17 03:05:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:07.544Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cpu=\"87\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545043910
Jul 17 03:05:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:07.558Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"promhttp_metric_handler_requests_total\", asset=\"twa-tsgx01\", asset_id=\"91\", code=\"200\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545043910
Jul 17 03:05:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:07.602Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", cpu=\"0\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Spark-msh-dellr740-olap003\", endpoint_id=\"13\", mode=\"steal\", module=\"NC-Spark\", module_id=\"47\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545046092
Jul 17 03:05:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:07.944Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"gpm.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689545033582
Jul 17 03:05:09 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:09.793Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_scanCount\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"assessment_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"91236b5787cc2a61a4ad1e613178dc07\", table=\"index_time_assessment_hos_bucket\", type=\"index\"}" timestamp=1689545048804
Jul 17 03:05:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:10.476Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_compactionsFailedCount\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"knowledge_base_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"4e53ab9d4bf390c647df30577df37a64\", table=\"index_time_knowledge_base_hos_bucket\", type=\"index\"}" timestamp=1689545048804
Jul 17 03:05:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:10.840Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_compactionsFailedCount\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"edc6ace358fe61cb62f74d0665ac6a0b\", table=\"firewall_hos_bucket\", type=\"data\"}" timestamp=1689545048804
Jul 17 03:05:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:11.279Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_numReferenceFiles\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"2f993a9cb121b0e389072da4f0f2eb0b\", table=\"troubleshooting_hos_bucket\", type=\"data\"}" timestamp=1689545050148
Jul 17 03:05:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:11.295Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_3_metric_scanSize_max\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545050148
Jul 17 03:05:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:11.307Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_minStoreFileAge\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"rtp_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"dddddd7aeb11a04db7628b214479089c\", table=\"index_time_rtp_hos_bucket\", type=\"index\"}" timestamp=1689545050148
Jul 17 03:05:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:11.608Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_DeleteBatch_min\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Server\"}" timestamp=1689545050148
Jul 17 03:05:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:11.625Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_storeFileSize\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", namespace=\"SYSTEM\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"ccc86592096566c6926dda82e28a082a\", table=\"LOG\", type=\"data\"}" timestamp=1689545050148
Jul 17 03:05:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:11.683Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_4_metric_majorCompactionTime_99_9th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545050148
Jul 17 03:05:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:11.740Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_session_record_hos_bucket_metric_majorCompactedOutputBytes\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545050148
Jul 17 03:05:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:11.767Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_193_1_metric_tableReadQueryPerSecond_mean_rate\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545050148
Jul 17 03:05:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:11.772Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_storeFileSize\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"knowledge_base_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"408298ac4baa4d15f68738efc36b7451\", table=\"index_filename_knowledge_base_hos_bucket\", type=\"index\"}" timestamp=1689545050148
Jul 17 03:05:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:11.785Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_TASK_metric_scanSize_95th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545050148
Jul 17 03:05:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:11.804Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_1_metric_tableWriteQueryPerSecond_count\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545050148
Jul 17 03:05:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:11.827Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_CheckAndDelete_num_ops\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Server\"}" timestamp=1689545050148
Jul 17 03:05:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:11.850Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_blockCacheTrailerMissCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Server\"}" timestamp=1689545050148
Jul 17 03:05:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:12.007Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_cpu_shares\", asset=\"pcap-tsgx08\", asset_id=\"87\", container=\"dashboard-metrics-scraper\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx08\", endpoint_id=\"481\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"1\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545048968
Jul 17 03:05:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:12.064Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_scanCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"knowledge_base_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"bf2a63dbae3b7c03de972a5d882a0894\", table=\"index_filename_knowledge_base_hos_bucket\", type=\"index\"}" timestamp=1689545050148
Jul 17 03:05:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:12.103Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_numBytesCompactedCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"default\", project=\"OLAP\", project_id=\"3\", region=\"fbe858613abcb26ec4b3f3d9ec686fd9\", table=\"index_partfile_default\", type=\"index\"}" timestamp=1689545050148
Jul 17 03:05:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:12.137Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_gc_duration_seconds\", asset=\"twa-dellr740-dt002\", asset_id=\"39\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"OLAP-Node-Exporter-twa-dellr740-dt002\", endpoint_id=\"259\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\", quantile=\"0.75\"}" timestamp=1689545052056
Jul 17 03:05:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:12.351Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_scaling_frequency_min_hertz\", asset=\"twa-dellr740-dt002\", asset_id=\"39\", cpu=\"0\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"OLAP-Node-Exporter-twa-dellr740-dt002\", endpoint_id=\"259\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545052056
Jul 17 03:05:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:12.356Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_3_metric_incrementTime_99th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545050148
Jul 17 03:05:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:12.562Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_194_2_metric_appendTime_num_ops\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545050148
Jul 17 03:05:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:12.652Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_frequency_min_hertz\", asset=\"twa-dellr740-dt002\", asset_id=\"39\", cpu=\"18\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"OLAP-Node-Exporter-twa-dellr740-dt002\", endpoint_id=\"259\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545052056
Jul 17 03:05:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:12.671Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_dormant\", asset=\"twa-dellr740-dt002\", asset_id=\"39\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"em3\", endpoint=\"OLAP-Node-Exporter-twa-dellr740-dt002\", endpoint_id=\"259\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545052056
Jul 17 03:05:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:12.848Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_tsg_table_report_result_metric_tableReadQueryPerSecond_15min_rate\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545050148
Jul 17 03:05:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:12.867Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_PostIndexUpdateFailureTime_95th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"PhoenixIndexer\"}" timestamp=1689545050148
Jul 17 03:05:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:12.926Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_filteredReadRequestCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"knowledge_base_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"d7545eddd6684b532e594f03edaf2b47\", table=\"index_time_knowledge_base_hos_bucket\", type=\"index\"}" timestamp=1689545050148
Jul 17 03:05:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:12.929Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_scaling_frequency_max_hertz\", asset=\"twa-dellr740-dt002\", asset_id=\"39\", cpu=\"27\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"OLAP-Node-Exporter-twa-dellr740-dt002\", endpoint_id=\"259\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545052056
Jul 17 03:05:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:12.947Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-dellr740-dt002\", asset_id=\"39\", cpu=\"5\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"OLAP-Node-Exporter-twa-dellr740-dt002\", endpoint_id=\"259\", mode=\"steal\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545052056
Jul 17 03:05:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:12.993Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_FUNCTION_metric_deleteTime_min\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545050148
Jul 17 03:05:13 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:13.002Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_scaling_frequency_min_hertz\", asset=\"twa-dellr740-dt002\", asset_id=\"39\", cpu=\"46\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"NC-Gohangout-twa-dellr740-dt002\", endpoint_id=\"179\", module=\"DC-Gohangout\", module_id=\"35\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545051525
Jul 17 03:05:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:15.359Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_packets_dropped_total\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"cadvisor\", interface=\"br_dign_c\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545043305
Jul 17 03:05:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:15.411Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_max_usage_bytes\", asset=\"twa-tsgx07\", asset_id=\"103\", container=\"ipmi-exporter\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545048880
Jul 17 03:05:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:15.688Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"twa-tsgx07\", asset_id=\"103\", condition=\"true\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-traefik-crd-l9hxr\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"6e3f5f94-9cad-4096-b668-32d84ce562eb\"}" timestamp=1689545048076
Jul 17 03:05:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:15.764Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_commands_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cmd=\"zremrangebyscore\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545046027
Jul 17 03:05:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:15.786Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"emergency.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"service\"}" timestamp=1689545047046
Jul 17 03:05:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:15.803Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_ulimits_soft\", asset=\"twa-tsgx07\", asset_id=\"103\", container=\"lb-tcp-443\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", ulimit=\"max_open_files\"}" timestamp=1689545038489
Jul 17 03:05:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:15.826Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"dashboard-metrics-scraper\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545048076
Jul 17 03:05:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:15.841Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_ready_replicas\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-0-firewall-f8d7c6884\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545048076
Jul 17 03:05:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:15.883Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_drop_total\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethe08168cb\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545041230
Jul 17 03:05:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:15.978Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_labels\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", job_name=\"helm-install-tsg-traffic-engine-numa-0\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545048076
Jul 17 03:05:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:16.112Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2did-scsi\\\\x2d362cea7f0848271002c25f03b9c8b41c5\\\\x2dpart4.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689545047046
Jul 17 03:05:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:16.261Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas_available\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"kubernetes-dashboard\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545048076
Jul 17 03:05:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:16.430Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"var-lib-kubelet-pods-64082a69\\\\x2d3be2\\\\x2d4b5e\\\\x2d9e3a\\\\x2dc0da16576d9e-volume\\\\x2dsubpaths-telegraf-telegraf\\\\x2dsecurity-0.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689545047046
Jul 17 03:05:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:16.495Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-sysusers.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689545047046
Jul 17 03:05:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:16.515Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"nfs-utils.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689545047046
Jul 17 03:05:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:16.536Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-veth6994e71f.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689545047046
Jul 17 03:05:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:16.693Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"21\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545041230
Jul 17 03:05:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:16.715Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db55\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545046027
Jul 17 03:05:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:16.747Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db86\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545046027
Jul 17 03:05:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:16.764Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"nsfs\", name=\"run-netns-cni\\\\x2d5de8a043\\\\x2dd390\\\\x2d01f3\\\\x2d0a25\\\\x2dd04a323aefee.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689545047046
Jul 17 03:05:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:16.785Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sshd-keygen@ecdsa.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"service\"}" timestamp=1689545047046
Jul 17 03:05:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:16.890Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx07\", asset_id=\"103\", container=\"redis\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Async\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545034710
Jul 17 03:05:18 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:18.892Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"clickhouse_os_guest_nice_time_cpu29\", asset=\"twa-dellr740-dt003\", asset_id=\"41\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"NC-Clickhouse-twa-dellr740-dt003\", endpoint_id=\"91\", module=\"NC-Clickhouse\", module_id=\"29\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545058300
Jul 17 03:05:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:19.477Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Tcp_Bypass_Pkt\", app_name=\"sapp4\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", vsysId=\"1\"}" timestamp=1689545053544
Jul 17 03:05:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:19.494Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-3ef5395f7954ec9793a826282b7ed847e3d60d4b1f102065c0de7ad629212003.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"scope\"}" timestamp=1689545044356
Jul 17 03:05:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:19.614Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db108\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545051998
Jul 17 03:05:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:19.660Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"49\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545057612
Jul 17 03:05:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:19.751Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_cpu_seconds_total\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-pod78246b9b_6c2f_4139_8e55_13e52335743d.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"slice\"}" timestamp=1689545044356
Jul 17 03:05:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:19.823Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link_mode\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"bond0\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545057612
Jul 17 03:05:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:19.842Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_compressed_total\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"eno1\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545057612
Jul 17 03:05:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:19.914Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db181\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545051998
Jul 17 03:05:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:19.982Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"20\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545057612
Jul 17 03:05:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:20.115Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"77\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545057612
Jul 17 03:05:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:20.306Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_data_received_bytes_total\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_2\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545057612
Jul 17 03:05:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:20.427Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"122\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545048263
Jul 17 03:05:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:20.482Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"44\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545057612
Jul 17 03:05:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:20.501Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"7\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545057612
Jul 17 03:05:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:20.509Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_file_descriptors\", asset=\"pcap-tsgx10\", asset_id=\"89\", container=\"telegraf-security\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689545055499
Jul 17 03:05:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:20.515Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_changes_total\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth814db6c2\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545057612
Jul 17 03:05:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:20.517Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_fifo_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth86dfd1bd\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545048263
Jul 17 03:05:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:20.573Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db119\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545051998
Jul 17 03:05:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:20.677Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_type\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"chart-values-configmap-reloader\", service=\"prometheus-kube-state-metrics\", type=\"Opaque\"}" timestamp=1689545053949
Jul 17 03:05:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:20.689Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"51\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545057612
Jul 17 03:05:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:20.702Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"91\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545048263
Jul 17 03:05:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:20.716Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Udp_Link_C2S\", app_name=\"sapp4\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", vsysId=\"1\"}" timestamp=1689545058534
Jul 17 03:05:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:20.725Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_hwmon_chip_names\", asset=\"pcap-tsgx10\", asset_id=\"89\", chip=\"0000:80:01_2_0000:82:00_2\", chip_name=\"bnxt_en\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545048263
Jul 17 03:05:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:20.736Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-4b0459ae\\\\x2daedd\\\\x2d4c07\\\\x2d81f2\\\\x2d0240bc0fb208-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2dtndd5.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689545050657
Jul 17 03:05:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:20.747Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_mtu_bytes\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens8f0np0\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545048263
Jul 17 03:05:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:20.760Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"6\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545048263
Jul 17 03:05:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:20.785Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_memory_Inactive_anon_bytes\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545048263
Jul 17 03:05:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:20.884Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db144\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545054840
Jul 17 03:05:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:20.964Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-pod26d0147a_fc67_4990_96ac_7d6c8e980d26.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"slice\"}" timestamp=1689545050657
Jul 17 03:05:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:21.031Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", container=\"dign-server-dns\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Async\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545050718
Jul 17 03:05:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:21.044Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_scrape_collector_duration_seconds\", asset=\"pcap-tsgx10\", asset_id=\"89\", collector=\"os\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545048263
Jul 17 03:05:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:21.069Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_packets_dropped_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"cadvisor\", interface=\"eno1\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545055659
Jul 17 03:05:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:21.084Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_state_started\", asset=\"pcap-tsgx10\", asset_id=\"89\", container=\"redis\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"cm-policy-local-cache-redis-master-69967b5879-8s2nz\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"efdb7e93-9e91-4d51-b008-336f3a7336fb\"}" timestamp=1689545053828
Jul 17 03:05:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:21.104Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_physical_state_id\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"mlx5_1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545048263
Jul 17 03:05:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:21.170Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"50\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545048263
Jul 17 03:05:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:21.187Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"k3s-orphaned-pods-dir-clean.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"service\"}" timestamp=1689545050657
Jul 17 03:05:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:21.220Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_hwmon_sensor_label\", asset=\"pcap-tsgx10\", asset_id=\"89\", chip=\"pci0000:00_0000:00:19_3\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", label=\"tctl\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", sensor=\"temp1\", service=\"prometheus-node-exporter\"}" timestamp=1689545048263
Jul 17 03:05:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:21.244Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_address_assign_type\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"eno35np2\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545048263
Jul 17 03:05:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:21.260Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_down_changes_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"eno35np2\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545048263
Jul 17 03:05:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:21.527Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_restart_policy\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-traefik-crd-tpdvt\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", type=\"OnFailure\", uid=\"48cb73fe-f319-4cb6-948c-44756cef01f8\"}" timestamp=1689545053828
Jul 17 03:05:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:21.550Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db0\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545054840
Jul 17 03:05:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:22.241Z caller=dedupe.go:112 component=remote level=error remote_name=52db6f url=http://10.10.10.159:10090/nz-agent/prometheus/proxy/api/v1/write msg="non-recoverable error" count=1000 exemplarCount=0 err="server returned HTTP status 400 : out of order sample"
Jul 17 03:05:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:23.528Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_fifo_total\", asset=\"msh-dellr740-olap004\", asset_id=\"17\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"docker0\", endpoint=\"NC-Spark-msh-dellr740-olap004\", endpoint_id=\"15\", module=\"NC-Spark\", module_id=\"47\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545062253
Jul 17 03:05:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:23.630Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"msh-tsgx09\", asset_id=\"55\", condition=\"false\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-kube-state-metrics-6b768c7bf8-db9cd\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"1c96629c-a0c6-4e4c-8e97-b6bbe91aa6d9\"}" timestamp=1689545060068
Jul 17 03:05:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:23.714Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-ipmi-exporter-v8l8s\", project=\"TSG-OS\", project_id=\"7\", reason=\"UnexpectedAdmissionError\", service=\"prometheus-kube-state-metrics\", uid=\"a3ae0996-f357-4355-bdfb-2d634555c081\"}" timestamp=1689545060068
Jul 17 03:05:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:24.032Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"msh-tsgx09\", asset_id=\"55\", container=\"telegraf-security\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", failure_type=\"pgmajfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"container\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689545048606
Jul 17 03:05:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:24.115Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_free_bytes\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"shm\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/70c311b2ab55787e217d750eb890fd9b9301c05719bfb93f9ef9f98d4b4f36b7/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545054790
Jul 17 03:05:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:24.147Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_mtu_bytes\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth64615143\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545060160
Jul 17 03:05:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:24.168Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_info\", asset=\"pcap-tsgx12\", asset_id=\"117\", created_by_kind=\"ReplicaSet\", created_by_name=\"dashboard-metrics-scraper-5b94897dc9\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", host_ip=\"10.10.10.172\", host_network=\"false\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", node=\"localhost\", nz_agent_id=\"3\", pod=\"dashboard-metrics-scraper-5b94897dc9-hgw87\", pod_ip=\"10.42.0.31\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"b65f87c9-c665-4fc6-9a65-8ece7ccf5f5a\"}" timestamp=1689545051122
Jul 17 03:05:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:24.173Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-pod78d32c4f_349d_455c_932b_b75d141ec6e9.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"slice\"}" timestamp=1689545051102
Jul 17 03:05:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:24.180Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_up_changes_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"eno35np2\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545060160
Jul 17 03:05:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:24.204Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_protocol_type\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth9642bab9\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545060160
Jul 17 03:05:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:24.240Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_observed_generation\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"prometheus-alertmanager\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545051122
Jul 17 03:05:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:24.265Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"13\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545060160
Jul 17 03:05:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:24.318Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_working_set_bytes\", asset=\"msh-tsgx09\", asset_id=\"55\", container=\"exporter\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545055469
Jul 17 03:05:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:24.332Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_packets_total\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", interface=\"net1\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545054623
Jul 17 03:05:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:24.461Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"dashboard-metrics-scraper\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Sync\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545060471
Jul 17 03:05:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:24.504Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"scrape_samples_scraped\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_instance=\"10.42.0.40:8080\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545051122
Jul 17 03:05:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:24.595Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:40-0000:40:01.1-0000:41:00.1-infiniband-mlx5_5.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689545051102
Jul 17 03:05:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:24.632Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-70c311b2ab55787e217d750eb890fd9b9301c05719bfb93f9ef9f98d4b4f36b7.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"scope\"}" timestamp=1689545058416
Jul 17 03:05:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:24.748Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-ens6f1np1.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689545058416
Jul 17 03:05:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:24.994Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_owner\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", job_name=\"helm-install-mrzcpd-exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", owner_is_controller=\"<none>\", owner_kind=\"<none>\", owner_name=\"<none>\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545051122
Jul 17 03:05:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:25.447Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"16\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545060160
Jul 17 03:05:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:25.533Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link_mode\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens2f0np0\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545060160
Jul 17 03:05:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:25.615Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"20\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545060160
Jul 17 03:05:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:25.770Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"configmap-reloader\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Discard\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545063056
Jul 17 03:05:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:25.812Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:e0-0000:e0:03.1-0000:e2:00.0-net-ens8f0np0.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689545051102
Jul 17 03:05:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:25.822Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_labels\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", pod=\"tsg-traffic-engine-numa-1-firewall-cdbcd8f4f-k98cr\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"a27a1ebc-016c-4998-a3f3-85e865dfcd16\"}" timestamp=1689545051122
Jul 17 03:05:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:26.126Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_packets_dropped_total\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"cadvisor\", interface=\"eno33np0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545063825
Jul 17 03:05:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:26.127Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled_time\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-kube-state-metrics-6b768c7bf8-c5glh\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"1d1df77f-66ac-4156-8ea3-66e8a287970d\"}" timestamp=1689545058744
Jul 17 03:05:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:26.153Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_metadata_resource_version\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"chart-values-prometheus\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545058744
Jul 17 03:05:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:26.168Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"twa-tsgx09\", asset_id=\"107\", condition=\"Progressing\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"prometheus-alertmanager\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"unknown\"}" timestamp=1689545058744
Jul 17 03:05:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:26.190Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_created\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"ipmi-exporter-6f8464dd94-c9gkv\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"8abd475c-c69e-4e96-9ca6-1c2e74278a42\"}" timestamp=1689545058744
Jul 17 03:05:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:26.195Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_cpu_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-pod2edb1c6e_4f58_4c31_b8c5_28dfc8f91f4d.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"slice\"}" timestamp=1689545053907
Jul 17 03:05:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:26.238Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"19\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545055810
Jul 17 03:05:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:26.278Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_files_free\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"shm\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/3b544c1f445293dd307ccad294511c80c09fcafe3545fffed402ff339aafbf23/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545055810
Jul 17 03:05:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:26.321Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"35\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545055810
Jul 17 03:05:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:26.400Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_start_time\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-configmap-reloader-hq5hk\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"cd026eda-ed7f-456d-af88-a83999108711\"}" timestamp=1689545058744
Jul 17 03:05:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:26.420Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"packagekit.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689545053907
Jul 17 03:05:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:26.501Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_system_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", container=\"telegraf\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689545058081
Jul 17 03:05:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:26.594Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"38\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545055810
Jul 17 03:05:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:26.608Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_device_id\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens4f1np1\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545055810
Jul 17 03:05:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:26.667Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_info\", asset=\"twa-tsgx09\", asset_id=\"107\", container=\"metrics\", container_id=\"containerd://6dfb472b5c6f4e10ab26aea947c493f86c4b6a1735b0b3576b03366de4b17cd5\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", image=\"docker.io/bitnami/redis-exporter:1.50.0\", image_id=\"sha256:ae434d812b4136c2a777882d508b25f384be532ea3e3686b3836bec55cf7b1e8\", image_spec=\"docker.io/bitnami/redis-exporter:1.50.0\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"cm-policy-local-cache-redis-master-69967b5879-74qzp\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"23f406a6-0255-4082-b091-d1fe8552acb1\"}" timestamp=1689545058744
Jul 17 03:05:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:26.892Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_info\", asset=\"twa-tsgx09\", asset_id=\"107\", created_by_kind=\"ReplicaSet\", created_by_name=\"kubernetes-dashboard-66dd5b456\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", host_ip=\"10.10.30.169\", host_network=\"false\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", node=\"localhost\", nz_agent_id=\"3\", pod=\"kubernetes-dashboard-66dd5b456-bgg44\", pod_ip=\"10.42.0.72\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"c8850d1a-9bcd-4bf0-adad-91f75a60d9cc\"}" timestamp=1689545058744
Jul 17 03:05:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:26.929Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_compressed_total\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth45cfb0fe\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545055810
Jul 17 03:05:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:26.948Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_tasks_current\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dbus.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689545053907
Jul 17 03:05:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:27.018Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_swap\", asset=\"twa-tsgx09\", asset_id=\"107\", container=\"kube-multus\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545054672
Jul 17 03:05:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:27.033Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_ports\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_endpoint=\"prometheus-kube-state-metrics\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", port_name=\"http\", port_number=\"8080\", port_protocol=\"TCP\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545058744
Jul 17 03:05:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:27.050Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"27\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545055810
Jul 17 03:05:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:27.065Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"18\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545055810
Jul 17 03:05:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:27.084Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"ext4\", name=\"var-lib-kubelet-pods-16605650\\\\x2dcdf7\\\\x2d400d\\\\x2d9016\\\\x2d8c2dbe1159c9-volume\\\\x2dsubpaths-config\\\\x2dvolume-telegraf-2.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689545053907
Jul 17 03:05:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:27.109Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_netstat_Udp6_OutDatagrams\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545055810
Jul 17 03:05:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:27.129Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_address_assign_type\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"virtio_dign_s\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545055810
Jul 17 03:05:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:27.136Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_receive_switch_relay_errors_total\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_2\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545055810
Jul 17 03:05:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:27.160Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:20-0000:20:03.1-0000:21:00.1-net-ens2f1np1.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689545053907
Jul 17 03:05:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:27.546Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"rescue.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"idle\", type=\"service\"}" timestamp=1689545054114
Jul 17 03:05:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:27.571Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_status_completion_time\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", job_name=\"helm-delete-tsg-traffic-engine-numa-1\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545055682
Jul 17 03:05:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:27.634Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_storageclass_info\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", provisioner=\"rancher.io/local-path\", reclaim_policy=\"Delete\", service=\"prometheus-kube-state-metrics\", storageclass=\"local-path\", volume_binding_mode=\"WaitForFirstConsumer\"}" timestamp=1689545055682
Jul 17 03:05:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:27.647Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_mallocs_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545059446
Jul 17 03:05:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:27.653Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_down_changes_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens4f3np3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545062882
Jul 17 03:05:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:27.654Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"configmap-reloader\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Read\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545050430
Jul 17 03:05:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:27.662Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"4\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545062882
Jul 17 03:05:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:27.673Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_commands_duration_seconds_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cmd=\"config\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545059446
Jul 17 03:05:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:27.681Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"16\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545062882
Jul 17 03:05:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:27.699Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_address_assign_type\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth43bf70e1\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545062882
Jul 17 03:05:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:27.707Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_packets_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"eno35np2\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545062882
Jul 17 03:05:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:27.721Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"47\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545062882
Jul 17 03:05:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:27.735Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"116\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545062882
Jul 17 03:05:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:27.744Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_avail_bytes\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"shm\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/548b9df90489f2fd81fa58e2aad7dc918b5445dc4beaad5ef0f59d037d3a4d8a/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545062882
Jul 17 03:05:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:27.818Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"17\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545062882
Jul 17 03:05:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:27.854Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_sockstat_UDP_mem\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545062882
Jul 17 03:05:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:27.878Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"rpcbind.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689545054114
Jul 17 03:05:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:27.905Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"rpc-statd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689545054114
Jul 17 03:05:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:27.906Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"var-lib-kubelet-pods-78ab1f97\\\\x2d9ecb\\\\x2d47ed\\\\x2dae81\\\\x2d048055fb4556-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2d2mcsg.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689545054114
Jul 17 03:05:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:28.075Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_max_usage_bytes\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"prometheus-alertmanager\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545053671
Jul 17 03:05:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:28.084Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_changes_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth9db5012c\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545062882
Jul 17 03:05:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:28.405Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"loki_querier_tail_bytes_total\", agent_name=\"msh-kvm-nz001\", application=\"loki\", datacenter=\"MSH\", instance=\"127.0.0.1:13100\"}" timestamp=1689545068327
Jul 17 03:05:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:28.424Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_created\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"prometheus-alertmanager\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545055682
Jul 17 03:05:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:28.517Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db138\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545059446
Jul 17 03:05:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:28.523Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"dign-client\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Async\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545064496
Jul 17 03:05:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:28.535Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"124\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545062882
Jul 17 03:05:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:28.542Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"16\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545062882
Jul 17 03:05:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:28.562Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_forks_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545062882
Jul 17 03:05:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:28.576Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_device_id\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"bond0\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545062882
Jul 17 03:05:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:28.745Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_up_changes_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"eno33np0\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545062882
Jul 17 03:05:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:28.760Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"dign-client\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Sync\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545064496
Jul 17 03:05:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:28.777Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_ulimits_soft\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"prometheus-alertmanager\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", ulimit=\"max_open_files\"}" timestamp=1689545053671
Jul 17 03:05:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:28.893Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_fifo_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens4f1np1\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545062882
Jul 17 03:05:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:28.910Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"overlay\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-5fb5362806ec0d39a1d6157eb0d1a029861953ba1a051d1adcf4cce93a7f78af-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689545054114
Jul 17 03:05:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:28.958Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"redis\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Discard\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545056623
Jul 17 03:05:29 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:29.092Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"108\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545062882
Jul 17 03:05:29 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:29.298Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"pcap-tsgx02\", asset_id=\"75\", container=\"dashboard-metrics-scraper\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx02\", endpoint_id=\"469\", exported_job=\"cadvisor\", failure_type=\"pgfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"1\", project=\"TSG-OS\", project_id=\"7\", scope=\"hierarchy\"}" timestamp=1689545043265
Jul 17 03:05:29 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:29.317Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"pcap-tsgx02\", asset_id=\"75\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx02\", endpoint_id=\"469\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"1\", pod=\"helm-install-configmap-reloader-p588n\", project=\"TSG-OS\", project_id=\"7\", reason=\"Evicted\", service=\"prometheus-kube-state-metrics\", uid=\"4d252e40-f20a-4a74-a65d-86038637988c\"}" timestamp=1689545056163
Jul 17 03:05:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:31.476Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_load_average_10s\", asset=\"twa-tsgx03\", asset_id=\"95\", container=\"prometheus-pushgateway\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545048842
Jul 17 03:05:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:31.491Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_resident_memory_bytes\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"mrzcpd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689545066907
Jul 17 03:05:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:31.512Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2dpartuuid-49572a39\\\\x2df871\\\\x2d4f22\\\\x2dadb4\\\\x2d4022b6a2df22.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689545066907
Jul 17 03:05:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:31.539Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_SnapshotObserverSnapshotFetchTime_99th_percentile\", asset=\"twa-dellr740-dt002\", asset_id=\"39\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt002-Hmaster\", endpoint_id=\"349\", module=\"DC-HBase\", module_id=\"3\", name=\"Master\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Quotas\"}" timestamp=1689545069703
Jul 17 03:05:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:31.558Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"123\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545059093
Jul 17 03:05:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:31.591Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"114\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545059093
Jul 17 03:05:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:31.727Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"62\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545060939
Jul 17 03:05:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:31.779Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_constraint_errors_transmitted_total\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_2\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545060939
Jul 17 03:05:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:31.781Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_mapped_file\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"kubernetes-dashboard\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545044559
Jul 17 03:05:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:31.899Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_drop_total\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"vethf231ad83\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545060791
Jul 17 03:05:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:32.030Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_memory_HugePages_Total\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545060939
Jul 17 03:05:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:32.085Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens8f0np0\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545060939
Jul 17 03:05:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:32.192Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sshd-keygen@ed25519.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"service\"}" timestamp=1689545066907
Jul 17 03:05:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:32.212Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"jvm_memory_pool_bytes_used\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-Yarn-twa-dellr740-dt001-NodeManager\", endpoint_id=\"363\", module=\"DC-Yarn\", module_id=\"1\", nz_agent_id=\"3\", pool=\"PS Survivor Space\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545071354
Jul 17 03:05:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:32.260Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_mspan_sys_bytes\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545066438
Jul 17 03:05:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:32.287Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas_unavailable\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"tsg-traffic-engine-numa-0-firewall\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545070436
Jul 17 03:05:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:32.403Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"70\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545059093
Jul 17 03:05:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:32.469Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"119\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545059093
Jul 17 03:05:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:32.474Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_file_descriptors\", asset=\"twa-tsgx05\", asset_id=\"99\", container=\"monit-stream\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545053337
Jul 17 03:05:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:32.488Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_status_start_time\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", job_name=\"helm-install-tsg-traffic-engine-numa-0\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545064068
Jul 17 03:05:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:32.510Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"3\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545060939
Jul 17 03:05:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:32.534Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"48\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545060605
Jul 17 03:05:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:32.542Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-update-done.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"oneshot\", type=\"service\"}" timestamp=1689545061015
Jul 17 03:05:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:32.550Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", fstype=\"ext4\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/data/var/lib/kubelet/pods/9982afd0-5736-4dce-b764-2f7b0c30831a/volume-subpaths/config-volume/telegraf/1\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545060605
Jul 17 03:05:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:32.581Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_data_received_bytes_total\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_8\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545059093
Jul 17 03:05:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:32.598Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"81\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545059093
Jul 17 03:05:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:32.620Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2did-wwn\\\\x2d0x62cea7f0848285002c25c6341fb95721\\\\x2dpart5.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689545066907
Jul 17 03:05:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:32.663Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_packets_total\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"cnibr0\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545060605
Jul 17 03:05:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:32.822Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_reservation_limit_bytes\", asset=\"msh-tsgx05\", asset_id=\"67\", container=\"prometheus-server-configmap-reload\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545055836
Jul 17 03:05:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:32.828Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_bytes_total\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"cadvisor\", interface=\"eth0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545054406
Jul 17 03:05:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:32.881Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_cpu_shares\", asset=\"twa-tsgx03\", asset_id=\"95\", container=\"mrapm-device\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545063123
Jul 17 03:05:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:32.895Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_spec_strategy_rollingupdate_max_unavailable\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"dashboard-metrics-scraper\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545064068
Jul 17 03:05:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:33.030Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-0b9ffe046fcf4cd096979ad8d16ea55f93485c7dc1eb6bc4c95cc6dcd20a6e79-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689545061015
Jul 17 03:05:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:33.054Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_bytes_total\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth4eba9fba\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545060939
Jul 17 03:05:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:33.066Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"msh-tsgx05\", asset_id=\"67\", condition=\"Progressing\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"mrzcpd-exporter\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"false\"}" timestamp=1689545064068
Jul 17 03:05:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:33.097Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_rss\", asset=\"twa-tsgx05\", asset_id=\"99\", container=\"coredns\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545053148
Jul 17 03:05:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:33.119Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_colls_total\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth66897ea4\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545060939
Jul 17 03:05:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:33.141Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_limit_bytes\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"prometheus-alertmanager-configmap-reload\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545062009
Jul 17 03:05:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:33.174Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_up_changes_total\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens4f1np1\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545060605
Jul 17 03:05:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:33.192Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"promhttp_metric_handler_errors_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cause=\"gathering\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545060605
Jul 17 03:05:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:33.451Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"99\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545060791
Jul 17 03:05:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:33.538Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"tmpfs\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/var/lib/kubelet/pods/03082888-996f-401e-985d-3fa51721b5b3/volumes/kubernetes.io~projected/kube-api-access-9nstt\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545059093
Jul 17 03:05:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:33.592Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"msh-tsgx11\", asset_id=\"111\", condition=\"true\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-configmap-reloader-nzdfr\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"e6f96c8f-7d00-4688-84c4-916e97112e6b\"}" timestamp=1689545059276
Jul 17 03:05:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:33.594Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_cpu_seconds_total\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"rdma-ndd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689545061015
Jul 17 03:05:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:33.595Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_receive_switch_relay_errors_total\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_9\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545060791
Jul 17 03:05:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:33.642Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_file_descriptors\", asset=\"twa-tsgx05\", asset_id=\"99\", container=\"lb-tcp-443\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545052375
Jul 17 03:05:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:33.664Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_hwmon_temp_celsius\", asset=\"msh-tsgx05\", asset_id=\"67\", chip=\"0000:60:03_1_0000:63:00_2\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", sensor=\"temp1\", service=\"prometheus-node-exporter\"}" timestamp=1689545060939
Jul 17 03:05:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:33.664Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link_mode\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens8f1np1\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545060791
Jul 17 03:05:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:33.723Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"11\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545060939
Jul 17 03:05:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:33.779Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_heap_sys_bytes\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545060605
Jul 17 03:05:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:33.820Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_max_fds\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"exporter-systemd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689545062644
Jul 17 03:05:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:34.002Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"15\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545060605
Jul 17 03:05:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:34.083Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_colls_total\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth5cd72b5c\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545059093
Jul 17 03:05:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:34.171Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_threads\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"prometheus-alertmanager\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545054178
Jul 17 03:05:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:34.173Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_files_free\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda2\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", fstype=\"ext4\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/mnt/onie-boot\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545060605
Jul 17 03:05:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:34.196Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_threads_max\", asset=\"twa-tsgx05\", asset_id=\"99\", container=\"firewall\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689545056933
Jul 17 03:05:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:34.233Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth65e2c8d5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545060605
Jul 17 03:05:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:34.258Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_annotations\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"shaping-announce-port-tsg-traffic-engine-numa-1\", uid=\"2a2a5987-e75d-48ac-b2b2-ed79e0957959\"}" timestamp=1689545059894
Jul 17 03:05:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:34.307Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Tcp_Bypass_STM\", app_name=\"sapp4\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", vsysId=\"1\"}" timestamp=1689545062297
Jul 17 03:05:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:34.385Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"61\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545059093
Jul 17 03:05:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:34.410Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-7c738da552f1e2c9143a23b6bcf02898a6e03e6c0c4d2315e7eb48f14c90d390.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"scope\"}" timestamp=1689545062644
Jul 17 03:05:35 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:35.113Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"7\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545060605
Jul 17 03:05:35 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:35.196Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas_unavailable\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"traefik\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545059894
Jul 17 03:05:35 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:35.440Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"73\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545060605
Jul 17 03:05:35 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:35.481Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_init_container_status_running\", asset=\"twa-tsgx05\", asset_id=\"99\", container=\"install-multus-binary\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"kube-multus-ds-vmk6p\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"e3971f33-46bb-450e-a67b-8e9e2f2ec265\"}" timestamp=1689545059894
Jul 17 03:05:35 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:35.633Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_state_started\", asset=\"twa-tsgx05\", asset_id=\"99\", container=\"redis\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"cm-policy-local-cache-redis-master-69967b5879-vlgl8\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"e8ef10b6-3745-4720-b3f4-f80080df3d45\"}" timestamp=1689545059894
Jul 17 03:05:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:40.746Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_incrementCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"cdead6d6bac6f43a6b32d23108a1fb76\", table=\"proxy_hos_bucket\", type=\"data\"}" timestamp=1689545079759
Jul 17 03:05:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:40.759Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_ports\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_endpoint=\"prometheus-alertmanager\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", port_name=\"http\", port_number=\"9093\", port_protocol=\"TCP\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545080034
Jul 17 03:05:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:40.779Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_rate_bytes_per_second\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_3\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545071647
Jul 17 03:05:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:40.789Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_hbase_table_meta_metric_compactionInputFileCount_median\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545079759
Jul 17 03:05:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:41.071Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_10_193_1_metric_majorCompactionInputFileCount_num_ops\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545079759
Jul 17 03:05:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:41.413Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"49\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545071647
Jul 17 03:05:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:41.427Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx06\", asset_id=\"61\", cpu=\"86\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx06\", endpoint_id=\"429\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545067064
Jul 17 03:05:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:41.445Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_receive_remote_physical_errors_total\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_4\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545071647
Jul 17 03:05:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:41.945Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_10_194_1_metric_compactionInputSize_min\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545079759
Jul 17 03:05:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:42.531Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2dpath-pci\\\\x2d0000:81:00.0\\\\x2dscsi\\\\x2d0:2:0:0\\\\x2dpart4.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689545080671
Jul 17 03:05:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:42.541Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_spec_volumes_persistentvolumeclaims_info\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", persistentvolumeclaim=\"prometheus-server\", pod=\"prometheus-server-6ff784d758-z96dq\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"5b7cc14f-eef4-4809-a696-cfb75e06120f\", volume=\"storage-volume\"}" timestamp=1689545075154
Jul 17 03:05:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:42.559Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_info\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.tsg-traffic-engine-numa-0.v1\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545075154
Jul 17 03:05:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:42.746Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"33\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545071647
Jul 17 03:05:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:43.602Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_system_table_bucket_meta_metric_putBatchTime_90th_percentile\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545079759
Jul 17 03:05:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:43.943Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"jvm_memory_pool_bytes_committed\", asset=\"pcap-dellr740-dt002\", asset_id=\"33\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-Yarn-pcap-dellr740-dt002-NodeManager\", endpoint_id=\"359\", module=\"DC-Yarn\", module_id=\"1\", nz_agent_id=\"3\", pool=\"Code Cache\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545081133
Jul 17 03:05:44 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:44.073Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_file_descriptors\", asset=\"pcap-tsgx01\", asset_id=\"71\", container=\"kube-state-metrics\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545079075
Jul 17 03:05:44 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:44.073Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_spec_parallelism\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", job_name=\"helm-install-traefik\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545075154
Jul 17 03:05:44 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:44.148Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", cpu=\"82\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545076242
Jul 17 03:05:44 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:44.158Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas_available\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"traefik\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545075154
Jul 17 03:05:45 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:45.224Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_restarts_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", container=\"dign-client\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"dign-client-nmxzm\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"4d40e67b-cb75-462e-b6c7-c667d37b9d55\"}" timestamp=1689545075154
Jul 17 03:05:45 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:45.676Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_scaling_frequency_hertz\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", cpu=\"16\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Gohangout-msh-dellr740-dt002\", endpoint_id=\"167\", module=\"DC-Gohangout\", module_id=\"35\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545082624
Jul 17 03:05:46 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:46.844Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"msh-kvm-cm001\", asset_id=\"3\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"CM-Node-Exporter-msh-kvm-cm001\", endpoint_id=\"391\", module=\"CM-Node-Exporter\", module_id=\"75\", name=\"10\", nz_agent_id=\"3\", parent_asset=\"msh-dellr740-pve001\", parent_asset_id=\"1\", project=\"Central-Management\", project_id=\"5\", type=\"Processor\"}" timestamp=1689545084235
Jul 17 03:05:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:49.093Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_session_record_hos_bucket_metric_getTime_98th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545087438
Jul 17 03:05:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:49.481Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_readonly\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"shm\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/1288df725fcbfecd4fdbd8a0e3571cb004951dade8125880acc73bb9945c1a5a/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545081797
Jul 17 03:05:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:49.624Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_session_record_hos_bucket_metric_putTime_99_9th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545087438
Jul 17 03:05:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:49.704Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_putCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", namespace=\"tsg_galaxy\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"ce0fe72f29202c7873dcf268f8d734d3\", table=\"gtpc_knowledge_base\", type=\"data\"}" timestamp=1689545087438
Jul 17 03:05:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:49.710Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db75\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545086625
Jul 17 03:05:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:49.747Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"init.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"scope\"}" timestamp=1689545081057
Jul 17 03:05:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:49.783Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_2_metric_splitTime_99_9th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545087438
Jul 17 03:05:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:49.844Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_up_changes_total\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens1f1np1\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545081797
Jul 17 03:05:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:49.905Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_1_metric_compactionOutputSize_98th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545087438
Jul 17 03:05:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:49.954Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-eno35np2.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689545081057
Jul 17 03:05:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:49.964Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_io_current\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"overlay_0-398\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545083842
Jul 17 03:05:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:49.986Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens8f0np0\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545086709
Jul 17 03:05:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:50.001Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_reads_merged_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"overlay_0-93\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545083842
Jul 17 03:05:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:50.042Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_state_id\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_7\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545086709
Jul 17 03:05:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:50.709Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Succeeded\", pod=\"helm-install-cm-policy-local-cache-psfzz\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"7c29bf8b-1230-4207-b809-f30adfc812f5\"}" timestamp=1689545079668
Jul 17 03:05:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:50.741Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_packets_received_total\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_11\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545081797
Jul 17 03:05:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:50.808Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_hbase_table_rsgroup_metric_compactionOutputFileCount_num_ops\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545087438
Jul 17 03:05:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:50.820Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx10\", asset_id=\"109\", cpu=\"3\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545081797
Jul 17 03:05:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:50.892Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_last_seen\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"telegraf-security\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689545079072
Jul 17 03:05:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:50.910Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"configmap-reloader\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"running\"}" timestamp=1689545070376
Jul 17 03:05:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:50.963Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_bytes_total\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens6f0np0\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545081797
Jul 17 03:05:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:51.016Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_hbase_table_namespace_metric_compactionOutputFileCount_90th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545087438
Jul 17 03:05:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:51.051Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-eno33np0.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689545081057
Jul 17 03:05:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:51.154Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_compressed_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens6f1np1\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545086709
Jul 17 03:05:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:51.191Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_scrape_collector_success\", asset=\"twa-tsgx08\", asset_id=\"105\", collector=\"uname\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545086709
Jul 17 03:05:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:51.247Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"twa-tsgx10\", asset_id=\"109\", cpu=\"39\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545081797
Jul 17 03:05:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:51.289Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"scrape_duration_seconds\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_instance=\"10.42.0.110:9290\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"ipmi-exporter\"}" timestamp=1689545084468
Jul 17 03:05:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:51.342Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx10\", asset_id=\"109\", cpu=\"62\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545081797
Jul 17 03:05:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:51.553Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_created\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_endpoint=\"systemd-exporter\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545081568
Jul 17 03:05:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:51.557Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"twa-tsgx08\", asset_id=\"105\", container=\"telegraf\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", failure_type=\"pgmajfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"container\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689545086910
Jul 17 03:05:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:51.612Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"89\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545086709
Jul 17 03:05:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:51.625Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_labels\", asset=\"twa-tsgx08\", asset_id=\"105\", configmap=\"sce-tsg-traffic-engine-numa-1\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545081568
Jul 17 03:05:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:51.712Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_labels\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"chart-values-ipmi-exporter\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545079668
Jul 17 03:05:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:51.920Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_files\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"shm\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/96a9f56f8880670d84ba1753f731c63c62abdde31dae3ba4a5cb169300591055/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545086709
Jul 17 03:05:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:51.943Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_readonly\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"shm\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/57586ef2b12d3be3dde6eb77d147247c3bda45d0866adf6b2d51c2372d8963e5/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545086709
Jul 17 03:05:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:52.021Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sysstat-collect.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"service\"}" timestamp=1689545086554
Jul 17 03:05:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:52.428Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", cpu=\"26\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545089337
Jul 17 03:05:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:52.634Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", container=\"local-path-provisioner\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Async\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545081573
Jul 17 03:05:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:52.763Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db62\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545079367
Jul 17 03:05:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:52.894Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda5\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", fstype=\"ext4\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/var/lib/kubelet/pods/93005333-2bbb-4ce3-a6b6-a5b632269832/volume-subpaths/config-volume/firewall/1\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545089337
Jul 17 03:05:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:53.321Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_file_descriptors\", asset=\"msh-tsgx08\", asset_id=\"65\", container=\"ipmi-exporter\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545074539
Jul 17 03:05:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:53.363Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx08\", asset_id=\"65\", cpu=\"61\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545088328
Jul 17 03:05:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:53.378Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx08\", asset_id=\"65\", cpu=\"37\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545088328
Jul 17 03:05:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:53.389Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_load_average_10s\", asset=\"msh-tsgx08\", asset_id=\"65\", container=\"local-path-provisioner\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545068723
Jul 17 03:05:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:53.416Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-virtual-net-veth401fe2c4.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689545091179
Jul 17 03:05:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:53.417Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_gc_duration_seconds\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", quantile=\"0.25\", service=\"ipmi-exporter\"}" timestamp=1689545085070
Jul 17 03:05:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:53.439Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_drop_total\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"eno1\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545083391
Jul 17 03:05:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:53.478Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_writes_total\", asset=\"msh-tsgx08\", asset_id=\"65\", container=\"configmap-reloader\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545065665
Jul 17 03:05:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:53.612Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", container=\"prometheus-pushgateway\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"iowaiting\"}" timestamp=1689545079643
Jul 17 03:05:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:53.720Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-infiniband-umad3.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689545080884
Jul 17 03:05:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:53.844Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_hwmon_temp_celsius\", asset=\"msh-tsgx08\", asset_id=\"65\", chip=\"pci0000:00_0000:00:19_3\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", sensor=\"temp1\", service=\"prometheus-node-exporter\"}" timestamp=1689545088328
Jul 17 03:05:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:53.911Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2dpartlabel-TSG\\\\x2dOS\\\\x2dDATA.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689545091179
Jul 17 03:05:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:53.931Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-7808816026b48879d750f82bcdb63695b75f59401621f081b8f28fe078660615-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689545085496
Jul 17 03:05:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:53.948Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"rpc-statd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"forking\", type=\"service\"}" timestamp=1689545091179
Jul 17 03:05:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:53.949Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_flags\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth9a1a84d3\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545089337
Jul 17 03:05:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:53.959Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx12\", asset_id=\"113\", container=\"firewall\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Read\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689545069406
Jul 17 03:05:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:53.990Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-delete-tsg-traffic-engine-numa-1-8tjrj\", project=\"TSG-OS\", project_id=\"7\", reason=\"NodeLost\", service=\"prometheus-kube-state-metrics\", uid=\"9061244f-a573-4738-a662-2e823d1f8614\"}" timestamp=1689545088663
Jul 17 03:05:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:54.015Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_persistentvolume_capacity_bytes\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", persistentvolume=\"pvc-625cdf6c-e629-442c-9ba2-7d1e8c98f868\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545088950
Jul 17 03:05:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:54.030Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx08\", asset_id=\"65\", cpu=\"57\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545088328
Jul 17 03:05:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:54.042Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx12\", asset_id=\"113\", cpu=\"52\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545083391
Jul 17 03:05:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:54.047Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_drop_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth858c4e97\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545089337
Jul 17 03:05:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:54.327Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_working_set_bytes\", asset=\"msh-tsgx08\", asset_id=\"65\", container=\"configmap-reloader\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545065665
Jul 17 03:05:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:54.465Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx12\", asset_id=\"113\", container=\"coredns\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Sync\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545078668
Jul 17 03:05:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:54.500Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_spec_replicas\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"prometheus-alertmanager\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545088950
Jul 17 03:05:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:54.582Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_observed_generation\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"tsg-traffic-engine-numa-0-firewall\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545082058
Jul 17 03:05:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:54.610Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_drop_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens3f1np1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545079409
Jul 17 03:05:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:55.807Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_threads\", asset=\"pcap-tsgx07\", asset_id=\"85\", container=\"monit-stream\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545084707
Jul 17 03:05:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:55.894Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"cm-policy-local-cache-redis-master-69967b5879-mnc4l\", project=\"TSG-OS\", project_id=\"7\", reason=\"NodeLost\", service=\"prometheus-kube-state-metrics\", uid=\"ae4f02ae-808a-4f27-b376-d8b3a79b4ccd\"}" timestamp=1689545082058
Jul 17 03:05:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:56.296Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-3ea51dfa79416f0b9e03ba6e05714d9f52183b93a0b06199a74ab4203050eb61.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"scope\"}" timestamp=1689545083372
Jul 17 03:05:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:56.315Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-kernel-config.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689545083372
Jul 17 03:05:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:56.352Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_fifo_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"virtio_dign_s\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545085883
Jul 17 03:05:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:56.372Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"snmptrapd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"notify\", type=\"service\"}" timestamp=1689545083372
Jul 17 03:05:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:56.414Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_daemonset_metadata_generation\", asset=\"pcap-tsgx03\", asset_id=\"73\", daemonset=\"svclb-traefik-ea05df14\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545093389
Jul 17 03:05:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:56.637Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_daemonset_status_number_ready\", asset=\"pcap-tsgx03\", asset_id=\"73\", daemonset=\"svclb-traefik-ea05df14\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545093389
Jul 17 03:05:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:56.900Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_net_dev_group\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens3f0np0\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545085883
Jul 17 03:05:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:56.917Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_nf_conntrack_stat_insert_failed\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545085883
Jul 17 03:05:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:57.174Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dnf-makecache.timer\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"timer\"}" timestamp=1689545083372
Jul 17 03:05:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:57.318Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"32\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545085883
Jul 17 03:05:58 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:58.111Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-dellr740-dt003\", asset_id=\"9\", cpu=\"46\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-dt003\", endpoint_id=\"229\", mode=\"steal\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"hbase_master,clickhouse_data,kafka,gohangout,zookeeper,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545097407
Jul 17 03:05:58 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:58.132Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_memory_Committed_AS_bytes\", asset=\"msh-dellr740-dt003\", asset_id=\"9\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-dt003\", endpoint_id=\"229\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"hbase_master,clickhouse_data,kafka,gohangout,zookeeper,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545097407
Jul 17 03:05:58 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:05:58.603Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_hwmon_temp_max_celsius\", asset=\"msh-dellr740-dt003\", asset_id=\"9\", chip=\"platform_coretemp_0\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-dt003\", endpoint_id=\"229\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"hbase_master,clickhouse_data,kafka,gohangout,zookeeper,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\", sensor=\"temp16\"}" timestamp=1689545097407
Jul 17 03:06:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:00.368Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_metadata_generation\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"traefik-64f55bb67d\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545087425
Jul 17 03:06:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:00.409Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_threads\", asset=\"msh-tsgx01\", asset_id=\"51\", container=\"dign-server-dns\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545088559
Jul 17 03:06:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:01.164Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_drop_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens7f1np1\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545089083
Jul 17 03:06:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:01.191Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"110\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545089083
Jul 17 03:06:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:01.281Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_free_bytes\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"shm\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/adf38312f586f9789395221f46cd8b7f6a29e1378158d03549f9e4d136a25c13/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545089083
Jul 17 03:06:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:01.298Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"36\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545089083
Jul 17 03:06:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:01.311Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_netstat_Udp6_InErrors\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545089083
Jul 17 03:06:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:01.405Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_net_input_bytes_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545085071
Jul 17 03:06:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:01.504Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"103\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545089083
Jul 17 03:06:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:01.635Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:a0-0000:a0:03.1-0000:a1:00.1-net-ens6f1np1.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689545084879
Jul 17 03:06:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:01.638Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_max_fds\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"hasp_monitor.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689545084879
Jul 17 03:06:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:02.472Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_last_seen\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689545092025
Jul 17 03:06:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:02.524Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_mspan_sys_bytes\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545089269
Jul 17 03:06:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:02.542Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx02\", asset_id=\"63\", cpu=\"94\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545098318
Jul 17 03:06:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:03.063Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx02\", asset_id=\"63\", cpu=\"16\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545098318
Jul 17 03:06:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:03.075Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_exporter_last_scrape_error\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545089269
Jul 17 03:06:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:03.161Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_info\", asset=\"msh-tsgx02\", asset_id=\"63\", container=\"helm\", container_id=\"containerd://73558cc983ea869c6e738846c00d0c7c03665514ce5205e296fdb29a45d4dd8c\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", image=\"docker.io/rancher/klipper-helm:v0.8.0-build20230510\", image_id=\"sha256:6f42df210d7faa0b6eb56f5f28bb061930c7834af1e7dd4b3586108ec6b1485b\", image_spec=\"rancher/klipper-helm:v0.8.0-build20230510\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-tsg-traffic-engine-numa-1-lhncj\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"35ba78e1-f4f4-4fdd-97e3-b2303cb918ac\"}" timestamp=1689545091412
Jul 17 03:06:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:03.286Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_device_id\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth2d52e57c\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545098318
Jul 17 03:06:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:03.301Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db39\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545089269
Jul 17 03:06:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:03.416Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_threads\", asset=\"msh-tsgx02\", asset_id=\"63\", container=\"telegraf\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689545081815
Jul 17 03:06:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:03.509Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"nginx_vts_server_cache_total\", asset=\"msh-dellr740-olap002\", asset_id=\"13\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"Galaxy-gateway-nginx-msh-dellr740-olap002\", endpoint_id=\"45\", host=\"192.168.20.222\", module=\"Galaxy-gateway-nginx\", module_id=\"11\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", status=\"hit\"}" timestamp=1689545102935
Jul 17 03:06:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:03.539Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_other_sys_bytes\", asset=\"pcap-dellr740-dt002\", asset_id=\"33\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"OLAP-Node-Exporter-pcap-dellr740-dt002\", endpoint_id=\"253\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545102497
Jul 17 03:06:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:03.561Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_up\", asset=\"pcap-dellr740-dt002\", asset_id=\"33\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"vethbb70265\", endpoint=\"OLAP-Node-Exporter-pcap-dellr740-dt002\", endpoint_id=\"253\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545102497
Jul 17 03:06:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:03.620Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"msh-tsgx02\", asset_id=\"63\", condition=\"true\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-pushgateway-bd476bcf9-fcnk7\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"6c93731d-28e4-47e7-a070-2872ea9641a7\"}" timestamp=1689545091412
Jul 17 03:06:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:03.772Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx02\", asset_id=\"63\", cpu=\"19\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545098318
Jul 17 03:06:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:03.798Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_colls_total\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"eno2\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545098318
Jul 17 03:06:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:03.811Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db84\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545089269
Jul 17 03:06:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:03.892Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_processes\", asset=\"msh-tsgx02\", asset_id=\"63\", container=\"telegraf-security\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689545079965
Jul 17 03:06:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:04.443Z caller=dedupe.go:112 component=remote level=info remote_name=52db6f url=http://10.10.10.159:10090/nz-agent/prometheus/proxy/api/v1/write msg="Dropped sample for series that was not explicitly dropped via relabelling" ref=4436650
Jul 17 03:06:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:04.565Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"net_conntrack_dialer_conn_failed_total\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", dialer_name=\"113\", instance=\"10.10.20.159:10090\", reason=\"timeout\", type=\"global\"}" timestamp=1689545102989
Jul 17 03:06:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:06.516Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_mcache_inuse_bytes\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Spark-msh-dellr740-olap003\", endpoint_id=\"13\", module=\"NC-Spark\", module_id=\"47\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545106094
Jul 17 03:06:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:06.911Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"tmpfs\", name=\"var-lib-kubelet-pods-5ee8d190\\\\x2d8de7\\\\x2d4d53\\\\x2da103\\\\x2dd0969342cef5-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2dckbw4.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689545093582
Jul 17 03:06:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:06.957Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dm-event.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"service\"}" timestamp=1689545093582
Jul 17 03:06:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:07.226Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"twa-tsgx01\", asset_id=\"91\", condition=\"unknown\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"configmap-reloader-6c55bbc49f-wgmn2\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"20ad385d-cecf-4e8f-a602-c8ed8267c99d\"}" timestamp=1689545101227
Jul 17 03:06:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:07.239Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cpu=\"20\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545103910
Jul 17 03:06:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:07.314Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_down_changes_total\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth444895a4\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545103910
Jul 17 03:06:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:08.095Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_name_assign_type\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"eno36np3\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545103910
Jul 17 03:06:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:08.111Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_labels\", asset=\"twa-tsgx01\", asset_id=\"91\", configmap=\"prometheus-server\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545101227
Jul 17 03:06:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:08.121Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-9ff428ce6fab3c88bb76f4412c790fd93eef47852aeeabc77e6699d546fc5842-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689545093582
Jul 17 03:06:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:08.297Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_readonly\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda4\", endpoint=\"NC-Spark-msh-dellr740-olap003\", endpoint_id=\"13\", fstype=\"xfs\", module=\"NC-Spark\", module_id=\"47\", mountpoint=\"/\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545106094
Jul 17 03:06:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:08.601Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"tsg_master_fail_log_s\", app_name=\"tsg_master\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"traffic-engine\", line_name=\"INTERIM-SESSION-RECORD\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", table_name=\"sendlog\", vsysId=\"1\"}" timestamp=1689545097817
Jul 17 03:06:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:11.426Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_compactionsQueuedCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"troubleshooting_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"dd829c862a6157803d65b4e2d0e75ad1\", table=\"index_time_troubleshooting_hos_bucket\", type=\"index\"}" timestamp=1689545110147
Jul 17 03:06:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:11.704Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_194_4_metric_deleteBatchTime_median\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545110147
Jul 17 03:06:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:12.264Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Sink_ageOfLastAppliedOp_98th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Replication\"}" timestamp=1689545110147
Jul 17 03:06:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:12.284Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_194_4_metric_appendTime_99th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545110147
Jul 17 03:06:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:12.319Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"java_lang_MemoryPool_Valid\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"CMS Old Gen\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545110147
Jul 17 03:06:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:12.345Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_readRequestCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"assessment_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"c326970ff7d07c2f0727098ea5f6dedb\", table=\"index_time_assessment_hos_bucket\", type=\"index\"}" timestamp=1689545110147
Jul 17 03:06:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:12.367Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_readRequestCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", namespace=\"SYSTEM\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"b0d947844d6d153d8ca0086e16920347\", table=\"LOG\", type=\"data\"}" timestamp=1689545110147
Jul 17 03:06:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:12.411Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_CompactionTime_99th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Server\"}" timestamp=1689545110147
Jul 17 03:06:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:12.527Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_default_metric_splitTime_75th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545110147
Jul 17 03:06:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:12.547Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_194_2_metric_tableReadQueryPerSecond_count\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545110147
Jul 17 03:06:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:12.623Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_3_metric_flushTime_mean\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545110147
Jul 17 03:06:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:15.222Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"twa-tsgx07\", asset_id=\"103\", container=\"metrics\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"cadvisor\", failure_type=\"pgfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"hierarchy\"}" timestamp=1689545100848
Jul 17 03:06:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:15.239Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545101230
Jul 17 03:06:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:15.405Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_annotations\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"chart-values-traefik-crd\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545108076
Jul 17 03:06:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:15.621Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"twa-tsgx07\", asset_id=\"103\", condition=\"true\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-prometheus-vmmrn\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"9a25cfd7-ced4-44df-b397-80f3bbb69a79\"}" timestamp=1689545108076
Jul 17 03:06:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:15.646Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_receive_switch_relay_errors_total\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_10\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545101230
Jul 17 03:06:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:15.668Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_speed_bytes\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"eno36np3\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545101230
Jul 17 03:06:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:15.683Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"36\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545101230
Jul 17 03:06:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:15.704Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_avg_ttl_seconds\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db1\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545106027
Jul 17 03:06:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:15.708Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Tcp_Lost_STM\", app_name=\"sapp4\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", vsysId=\"1\"}" timestamp=1689545111690
Jul 17 03:06:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:15.771Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_labels\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", job_name=\"helm-install-tsg-traffic-engine-numa-0\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545108076
Jul 17 03:06:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:15.792Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_size_bytes\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", fstype=\"ext4\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/var\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545101230
Jul 17 03:06:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:15.805Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"ext4\", name=\"data-var-lib-kubelet-pods-64082a69\\\\x2d3be2\\\\x2d4b5e\\\\x2d9e3a\\\\x2dc0da16576d9e-volume\\\\x2dsubpaths-config\\\\x2dvolume-telegraf\\\\x2dsecurity-2.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689545107046
Jul 17 03:06:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:15.870Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_changes_total\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens1f1np1\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545101230
Jul 17 03:06:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:15.925Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_init_container_status_terminated\", asset=\"twa-tsgx07\", asset_id=\"103\", container=\"tsg-dign-init\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"dign-server-2l4bb\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"69414658-aafa-4782-9372-26c60a463de5\"}" timestamp=1689545108076
Jul 17 03:06:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:15.948Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_memory_FilePmdMapped_bytes\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545101230
Jul 17 03:06:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:15.978Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"twa-tsgx07\", asset_id=\"103\", container=\"telegraf\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", state=\"running\"}" timestamp=1689545105614
Jul 17 03:06:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:15.999Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"123\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545101230
Jul 17 03:06:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:16.774Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_memory_MemTotal_bytes\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545101230
Jul 17 03:06:18 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:18.977Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_avail_bytes\", asset=\"msh-tsgx04\", asset_id=\"57\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"shm\", endpoint=\"TSG-OS-msh-tsgx04\", endpoint_id=\"425\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/04a54dc416d4b1619b786e192d03ab2698c90a3ab9fa09bfd7698550de41e618/shm\", nz_agent_id=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545104557
Jul 17 03:06:18 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:18.995Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"msh-tsgx04\", asset_id=\"57\", cpu=\"120\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx04\", endpoint_id=\"425\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545104557
Jul 17 03:06:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:19.752Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"clickhouse_replicated_checks\", asset=\"twa-dellr740-dt003\", asset_id=\"41\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"NC-Clickhouse-twa-dellr740-dt003\", endpoint_id=\"91\", module=\"NC-Clickhouse\", module_id=\"29\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545118301
Jul 17 03:06:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:20.290Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_link_downed_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"mlx5_8\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545108263
Jul 17 03:06:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:20.328Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"var-lib-kubelet-pods-10741627\\\\x2d0ed0\\\\x2d416d\\\\x2dac50\\\\x2d5a8214dc1db7-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2dqbwc2.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689545110657
Jul 17 03:06:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:20.536Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_free_bytes\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"shm\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/2e76363666cd4185ab33c56431d3ebe22fe18788f7c5d7c2329171bd08c5f2ca/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545108263
Jul 17 03:06:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:20.708Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", container=\"metrics-server\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"cadvisor\", failure_type=\"pgmajfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"hierarchy\"}" timestamp=1689545104968
Jul 17 03:06:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:20.734Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_annotations\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", job_name=\"helm-install-traefik\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545113828
Jul 17 03:06:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:20.767Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"pcap-tsgx10\", asset_id=\"89\", condition=\"Available\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"cm-policy-local-cache-redis-master\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"unknown\"}" timestamp=1689545113828
Jul 17 03:06:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:20.870Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"95\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545108263
Jul 17 03:06:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:20.893Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"scrape_series_added\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_instance=\"10.42.0.129:9273\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", vsysId=\"1\"}" timestamp=1689545118534
Jul 17 03:06:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:20.937Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_created\", asset=\"pcap-tsgx10\", asset_id=\"89\", configmap=\"kube-root-ca.crt\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545113828
Jul 17 03:06:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:20.941Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_spec_strategy_rollingupdate_max_surge\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"prometheus-pushgateway\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545113828
Jul 17 03:06:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:20.962Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545108263
Jul 17 03:06:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:20.988Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545108263
Jul 17 03:06:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:21.031Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_files_free\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"shm\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/aebe77d3b5b78e4f598e7b51fd01ec5ef6e9c50bb0cb2dc1010f3ecfe3e703a8/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545108263
Jul 17 03:06:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:21.070Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-veth8d94c437.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689545110657
Jul 17 03:06:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:21.261Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failcnt\", asset=\"twa-tsgx06\", asset_id=\"101\", container=\"redis\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545098183
Jul 17 03:06:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:21.286Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"eno36np3\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545108263
Jul 17 03:06:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:21.306Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_compressed_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens4f2np2\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545108263
Jul 17 03:06:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:21.336Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"lvm2-monitor.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"service\"}" timestamp=1689545104356
Jul 17 03:06:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:21.355Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethdd9c27e1\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545117612
Jul 17 03:06:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:21.371Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_slowlog_last_id\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545111998
Jul 17 03:06:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:21.392Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"nvmefc-boot-connections.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"service\"}" timestamp=1689545104356
Jul 17 03:06:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:21.411Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"74\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545108263
Jul 17 03:06:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:21.782Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"36\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545108263
Jul 17 03:06:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:21.804Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_labels\", asset=\"twa-tsgx06\", asset_id=\"101\", configmap=\"kube-apiserver-legacy-service-account-token-tracking\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545113949
Jul 17 03:06:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:21.826Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"shm\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/448292d42a9d5cd3469bea891dd4ffd146819eb961b88336c45cf0bad074b750/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545117612
Jul 17 03:06:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:21.850Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_protocol_type\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens6f0np0\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545117612
Jul 17 03:06:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:21.871Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_oom_events_total\", asset=\"twa-tsgx06\", asset_id=\"101\", container=\"telegraf\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689545106038
Jul 17 03:06:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:22.025Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_drop_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth4c680714\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545108263
Jul 17 03:06:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:22.027Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Unknown\", pod=\"traefik-64f55bb67d-v57x6\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"eb9c8310-844c-4b72-9e0c-d86e20973848\"}" timestamp=1689545113828
Jul 17 03:06:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:22.086Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_info\", asset=\"pcap-tsgx10\", asset_id=\"89\", container=\"mrapm-device\", container_id=\"containerd://5e13c3a0d5f9065ec9ae00372c26ba8786248c86182de51d1ae7eb8dffed8552\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", image=\"registry.gdnt-cloud.website/mrzcpd-exporter:1.0.1\", image_id=\"sha256:b1a7adfb61be24490a1321c37405d86204b78b1ee251ed959d48dc357310fb1c\", image_spec=\"registry.gdnt-cloud.website/mrzcpd-exporter:1.0.1\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"mrzcpd-exporter-5cf7f967cc-t58mg\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"eafb2c24-f0a2-46e9-837e-e7db0ce25c25\"}" timestamp=1689545113828
Jul 17 03:06:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:22.280Z caller=dedupe.go:112 component=remote level=error remote_name=52db6f url=http://10.10.10.159:10090/nz-agent/prometheus/proxy/api/v1/write msg="non-recoverable error" count=1000 exemplarCount=0 err="server returned HTTP status 400 : out of order sample"
Jul 17 03:06:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:22.326Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"jvm_memory_max_bytes\", application=\"galaxy-job-executor\", area=\"nonheap\", asset=\"msh-dellr740-olap002\", asset_id=\"13\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"Galaxy-job-executor-msh-dellr740-olap002\", endpoint_id=\"53\", id=\"Compressed Class Space\", module=\"Galaxy-job-executor\", module_id=\"19\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545121271
Jul 17 03:06:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:22.364Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_ips\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", ip=\"10.42.0.110\", ip_family=\"4\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-cm-policy-local-cache-x5lq4\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"5859e672-b6bc-47e9-99af-51cf757d48a4\"}" timestamp=1689545113828
Jul 17 03:06:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:22.449Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"up\", asset=\"msh-dellr740-olap002\", asset_id=\"13\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"Galaxy-job-executor-msh-dellr740-olap002\", endpoint_id=\"53\", instance=\"192.168.20.222:8185\", job=\"53\", module=\"Galaxy-job-executor\", module_id=\"19\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545121271
Jul 17 03:06:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:22.582Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_init_container_info\", asset=\"pcap-tsgx10\", asset_id=\"89\", container=\"tsg-dign-init\", container_id=\"containerd://c59a091296fe59a3dbcaea7608e0044cda06b1b47c046395ca478739453f9fde\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", image=\"docker.io/library/dign-server-dns:latest\", image_id=\"sha256:6987ac49c2d264e9289120b41e4a48536092ce6fbe41c981cdd4ad115feae8a4\", image_spec=\"dign-server-dns:latest\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"dign-server-55b4f\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"e12ed51a-28b1-413a-82be-8a0fcac5cffa\"}" timestamp=1689545113828
Jul 17 03:06:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:22.584Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_init_container_status_running\", asset=\"pcap-tsgx10\", asset_id=\"89\", container=\"firewall-init\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", pod=\"tsg-traffic-engine-numa-1-firewall-5d7bfbbbfb-4wgxr\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"ab53e5a7-f22e-498d-b5af-535ae7359350\"}" timestamp=1689545113828
Jul 17 03:06:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:24.129Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"coredns\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Write\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545121455
Jul 17 03:06:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:24.156Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"57\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545120160
Jul 17 03:06:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:24.390Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"abrt-xorg.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689545111102
Jul 17 03:06:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:24.443Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"configmap-reloader\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Discard\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545108833
Jul 17 03:06:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:24.466Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas_ready\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"prometheus-pushgateway\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545111122
Jul 17 03:06:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:24.574Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_errs_total\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"eno35np2\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545114790
Jul 17 03:06:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:24.704Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens4f1np1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545120160
Jul 17 03:06:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:24.728Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_up\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens7f1np1\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545114790
Jul 17 03:06:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:24.744Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sysstat-collect.timer\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"timer\"}" timestamp=1689545118416
Jul 17 03:06:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:24.761Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"var-lib-kubelet-pods-58676ba8\\\\x2d2676\\\\x2d43fd\\\\x2db12b\\\\x2d038d471289b5-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2dgnz6t.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689545118416
Jul 17 03:06:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:24.896Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_spec_type\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", type=\"ClusterIP\", uid=\"0afa518f-edd9-4640-9268-a8b5eb21846a\"}" timestamp=1689545120068
Jul 17 03:06:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:24.911Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"76\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545114790
Jul 17 03:06:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:24.918Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"msh-tsgx09\", asset_id=\"55\", condition=\"false\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-alertmanager-6df7f457b6-ct85d\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"e335f8b4-74cf-439e-9760-c54f88f85a5d\"}" timestamp=1689545120068
Jul 17 03:06:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:24.960Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_spec_replicas\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"cm-policy-local-cache-redis-master-69967b5879\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545120068
Jul 17 03:06:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:24.976Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failcnt\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"dign-server-web\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545119523
Jul 17 03:06:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:24.979Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"27\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545114790
Jul 17 03:06:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:24.987Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"56\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545120160
Jul 17 03:06:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:25.025Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_created\", asset=\"msh-tsgx09\", asset_id=\"55\", configmap=\"kube-root-ca.crt\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-node-lease\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545120068
Jul 17 03:06:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:25.059Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_module_fork_last_cow_size\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545121795
Jul 17 03:06:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:25.098Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_usage_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"prometheus-alertmanager\", cpu=\"total\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545118822
Jul 17 03:06:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:25.108Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas_available\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"prometheus-pushgateway\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545120068
Jul 17 03:06:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:25.151Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"auth-rpcgss-module.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"service\"}" timestamp=1689545118416
Jul 17 03:06:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:25.189Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_limit_bytes\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"dign-server-web\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545123419
Jul 17 03:06:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:25.222Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_read_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda2\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545119071
Jul 17 03:06:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:25.228Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-podf7af3cd7_b972_4aba_a12a_7b5c827e57eb.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"slice\"}" timestamp=1689545111102
Jul 17 03:06:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:25.234Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_discards_transmitted_total\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_11\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545114790
Jul 17 03:06:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:25.367Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545120160
Jul 17 03:06:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:25.398Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"0\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545120160
Jul 17 03:06:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:25.428Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_cpu_shares\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545118586
Jul 17 03:06:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:25.443Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-f76f3e607f7624715c243ab6d2d8410c2efb5a74dd1c2fb9c213720c63dd55d2-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689545118416
Jul 17 03:06:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:25.460Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_writes_merged_total\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/run\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545117616
Jul 17 03:06:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:25.473Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Succeeded\", pod=\"helm-install-systemd-exporter-2h74t\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"75668a8f-85fc-45c3-9b22-ebd6bba21c17\"}" timestamp=1689545120068
Jul 17 03:06:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:25.696Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-virtual-net-veth168605c4.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689545111102
Jul 17 03:06:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:26.549Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx09\", asset_id=\"107\", container=\"dign-server-web\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Sync\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545115361
Jul 17 03:06:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:26.955Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"125\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545115810
Jul 17 03:06:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:27.040Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"125\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545115810
Jul 17 03:06:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:27.143Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_mapped_file\", asset=\"twa-tsgx09\", asset_id=\"107\", container=\"telegraf-security\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689545121157
Jul 17 03:06:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:27.312Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-acc5fcbf\\\\x2dfcf1\\\\x2d426e\\\\x2db97e\\\\x2d6d5fa645e542-volume\\\\x2dsubpaths-config\\\\x2dpath-dign\\\\x2dserver\\\\x2ddns-1.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689545113907
Jul 17 03:06:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:27.388Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:40-0000:40:01.1-0000:41:00.1-infiniband_mad-umad5.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689545113907
Jul 17 03:06:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:27.427Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_metadata_generation\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"kubernetes-dashboard\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545118744
Jul 17 03:06:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:27.452Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_virtual_memory_bytes\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"gpm.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689545113907
Jul 17 03:06:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:27.479Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_info\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_endpoint=\"prometheus-alertmanager\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545118744
Jul 17 03:06:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:27.486Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"ipmi_up\", asset=\"msh-tsgx10\", asset_id=\"69\", collector=\"ipmi\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"ipmi-exporter\"}" timestamp=1689545121323
Jul 17 03:06:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:27.487Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_last_terminated_reason\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"prometheus-pushgateway\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-pushgateway-bd476bcf9-jkpjd\", project=\"TSG-OS\", project_id=\"7\", reason=\"Unknown\", service=\"prometheus-kube-state-metrics\", uid=\"7ac64d36-f972-409d-8e70-7eed1da8b973\"}" timestamp=1689545115682
Jul 17 03:06:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:27.506Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"102\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545115810
Jul 17 03:06:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:27.539Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_bytes_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", interface=\"ens8f0np0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545124341
Jul 17 03:06:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:27.571Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"dashboard-metrics-scraper\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Total\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545111246
Jul 17 03:06:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:27.590Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_status_start_time\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", job_name=\"helm-install-traefik\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545115682
Jul 17 03:06:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:27.624Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_files_free\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"shm\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/8a8a40fbdb832835f96612d7f1a8f30a2083597b0e95502cb454c1417656feb0/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545122882
Jul 17 03:06:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:27.659Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_annotations\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\", uid=\"0b4c453b-217b-4bf4-a7a7-d77ef726d0e4\"}" timestamp=1689545115682
Jul 17 03:06:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:27.678Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_multicast_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"virtio_dign_s\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545122882
Jul 17 03:06:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:27.699Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"13\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545122882
Jul 17 03:06:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:27.756Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_threads_max\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"kubernetes-dashboard\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545109270
Jul 17 03:06:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:27.763Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"17\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545115810
Jul 17 03:06:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:27.775Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_free_bytes\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"shm\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/3b2f652db0714906f7218b339fcd157b32dbc7432e2f7e2b52b5bacfea6fc297/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545122882
Jul 17 03:06:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:27.790Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-d49e1f92ac144fe07247560cf84acd290693f407eceaa245acdd5295cccc7637-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689545113907
Jul 17 03:06:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:27.820Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx09\", asset_id=\"107\", container=\"dign-server-dns\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Discard\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545115116
Jul 17 03:06:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:27.844Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"48\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545115810
Jul 17 03:06:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:27.869Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_vl15_dropped_total\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_9\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545115810
Jul 17 03:06:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:27.882Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"tmpfs\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/var/lib/kubelet/pods/7ac64d36-f972-409d-8e70-7eed1da8b973/volumes/kubernetes.io~projected/kube-api-access-8gkw2\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545122882
Jul 17 03:06:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:27.902Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_memory_Cached_bytes\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545115810
Jul 17 03:06:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:27.925Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:80-0000:80:01.2-0000:82:00.1-net-ens4f1np1.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689545114114
Jul 17 03:06:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:27.927Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_mtu_bytes\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth002839a7\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545122882
Jul 17 03:06:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:27.944Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_multicast_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth002839a7\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545122882
Jul 17 03:06:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:27.991Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db165\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545119446
Jul 17 03:06:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:28.075Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_changes_total\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth0fc63319\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545115810
Jul 17 03:06:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:28.119Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_local_link_integrity_errors_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_2\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545122882
Jul 17 03:06:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:28.159Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"70\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545122882
Jul 17 03:06:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:28.160Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_read_seconds_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/run\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545124341
Jul 17 03:06:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:28.179Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled_time\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-pushgateway-bd476bcf9-jkpjd\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"7ac64d36-f972-409d-8e70-7eed1da8b973\"}" timestamp=1689545115682
Jul 17 03:06:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:28.183Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db131\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545119446
Jul 17 03:06:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:28.183Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545122882
Jul 17 03:06:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:28.200Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_fully_labeled_replicas\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-0-firewall-7cbb5bf99c\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545115682
Jul 17 03:06:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:28.205Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_commands_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cmd=\"zscore\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545119446
Jul 17 03:06:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:28.226Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"11\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545115810
Jul 17 03:06:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:28.240Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"55\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545115810
Jul 17 03:06:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:28.267Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-tmpfiles-setup-dev.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689545114114
Jul 17 03:06:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:28.273Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"slice\"}" timestamp=1689545114114
Jul 17 03:06:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:28.360Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_tasks_current\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"libstoragemgmt.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689545114114
Jul 17 03:06:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:28.492Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"101\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545122882
Jul 17 03:06:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:28.658Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_info\", asset=\"msh-tsgx10\", asset_id=\"69\", configmap=\"shaping-tsg-traffic-engine-numa-0\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545115682
Jul 17 03:06:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:28.674Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_persistentvolume_status_phase\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", persistentvolume=\"pvc-6ef68438-8e4b-4575-8011-dff4f7fefea1\", phase=\"Pending\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545115682
Jul 17 03:06:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:28.693Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_annotations\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-pushgateway\", uid=\"b9d0ffc2-21d5-4629-9a18-c5637ee5c566\"}" timestamp=1689545115682
Jul 17 03:06:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:28.715Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"shm\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/3b2f652db0714906f7218b339fcd157b32dbc7432e2f7e2b52b5bacfea6fc297/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545122882
Jul 17 03:06:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:28.776Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"scrape_samples_post_metric_relabeling\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_instance=\"10.10.20.170:9100\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545122882
Jul 17 03:06:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:28.797Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_down_changes_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth476a8450\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545122882
Jul 17 03:06:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:28.812Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_scrape_collector_success\", asset=\"msh-tsgx10\", asset_id=\"69\", collector=\"textfile\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545122882
Jul 17 03:06:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:28.826Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db119\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545119446
Jul 17 03:06:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:28.842Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"45\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545122882
Jul 17 03:06:29 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:29.181Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"cortex_kv_request_duration_seconds_count\", agent_name=\"msh-kvm-nz001\", application=\"loki\", datacenter=\"MSH\", instance=\"127.0.0.1:13100\", kv_name=\"distributor-lifecycler\", operation=\"CAS\", role=\"primary\", status_code=\"200\", type=\"memberlist\"}" timestamp=1689545128331
Jul 17 03:06:29 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:29.199Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_heap_sys_bytes\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545122882
Jul 17 03:06:29 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:29.236Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"cadvisor_version_info\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", kernelVersion=\"5.17.15-1.el8.x86_64\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", osVersion=\"TSG-OS v23.07-rc1-079e964 (TSGXNXR620G40R01P0906)\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545124841
Jul 17 03:06:30 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:30.971Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"119\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545120939
Jul 17 03:06:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:31.006Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_info\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"chart-values-traefik-crd\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545124068
Jul 17 03:06:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:31.032Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda5\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", fstype=\"ext4\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/data/var/lib/kubelet/pods/99e7b7f0-307f-4274-a670-faf429f178f7/volume-subpaths/telegraf/telegraf/0\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545120939
Jul 17 03:06:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:31.060Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"nginx_vts_main_connections\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"Galaxy-hos-nginx-msh-dellr740-dt001\", endpoint_id=\"201\", module=\"Galaxy-hos-nginx\", module_id=\"13\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", status=\"writing\"}" timestamp=1689545130872
Jul 17 03:06:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:31.067Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_free_bytes\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"shm\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/1f9bf43eaab828aaaf3e781dba021e066218f2fca2ebfd2fc38feb4ec233a4e2/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545120939
Jul 17 03:06:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:31.078Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_files_free\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"shm\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/e59e7b87fde78e7542641071d0198180142793becfd47a65488a32d495401f78/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545120939
Jul 17 03:06:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:31.123Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"124\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545120939
Jul 17 03:06:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:31.215Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_oom_events_total\", asset=\"msh-tsgx05\", asset_id=\"67\", container=\"metrics-server\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545109543
Jul 17 03:06:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:31.223Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"62\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545120939
Jul 17 03:06:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:31.233Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545120939
Jul 17 03:06:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:31.269Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"115\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545120939
Jul 17 03:06:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:31.298Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"25\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545120939
Jul 17 03:06:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:31.315Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"msh-tsgx05\", asset_id=\"67\", condition=\"true\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"coredns-696d987597-cgfrx\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"84148aa3-78e1-49bc-9b4c-b596d7dd94b8\"}" timestamp=1689545124068
Jul 17 03:06:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:31.335Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-configmap-reloader-xwkxx\", project=\"TSG-OS\", project_id=\"7\", reason=\"UnexpectedAdmissionError\", service=\"prometheus-kube-state-metrics\", uid=\"e2e0cb44-6f39-4de2-b32f-85c91a6fc2b9\"}" timestamp=1689545124068
Jul 17 03:06:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:31.394Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-f3fbbd5c6b2d46b94138f71c84c8bc38d61733259abcabc52124f8404c386984-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689545121015
Jul 17 03:06:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:31.439Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_limit_bytes\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/shm\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545119516
Jul 17 03:06:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:31.445Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_up\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens6f0np0\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545119093
Jul 17 03:06:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:31.447Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"twa-tsgx03\", asset_id=\"95\", condition=\"unknown\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"metrics-server-54dc485875-dndjr\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"3d844771-7231-4073-bad4-00bcb03c68e0\"}" timestamp=1689545130436
Jul 17 03:06:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:31.455Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_io_time_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda2\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545119516
Jul 17 03:06:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:31.456Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"77\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545119093
Jul 17 03:06:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:31.464Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"70\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545119093
Jul 17 03:06:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:31.469Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_up\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"bond0\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545119093
Jul 17 03:06:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:31.472Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"lvm2-lvmpolld.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"service\"}" timestamp=1689545126907
Jul 17 03:06:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:31.478Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_carrier_total\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"eno34np1\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545119093
Jul 17 03:06:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:31.494Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:20-0000:20:03.1-0000:21:00.0-infiniband_mad-umad2.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689545126907
Jul 17 03:06:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:31.500Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db176\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545126438
Jul 17 03:06:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:31.536Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"1\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545119093
Jul 17 03:06:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:31.550Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_reads_total\", asset=\"twa-tsgx03\", asset_id=\"95\", container=\"redis\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545109599
Jul 17 03:06:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:31.577Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dracut-mount.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"service\"}" timestamp=1689545126907
Jul 17 03:06:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:31.600Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db71\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545126438
Jul 17 03:06:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:31.601Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"72\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545119093
Jul 17 03:06:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:31.622Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2duuid-b8687430\\\\x2d07ac\\\\x2d4ffa\\\\x2d85d8\\\\x2d4e7bd9e0b1fb.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689545126907
Jul 17 03:06:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:31.627Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-kube-state-metrics-6b768c7bf8-8z25n\", project=\"TSG-OS\", project_id=\"7\", reason=\"Evicted\", service=\"prometheus-kube-state-metrics\", uid=\"0f76af51-4cb4-40c5-8ff5-c445f8b7b608\"}" timestamp=1689545130436
Jul 17 03:06:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:31.663Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"123\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545119093
Jul 17 03:06:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:31.691Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas_available\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"local-path-provisioner\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545119276
Jul 17 03:06:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:31.710Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"msh-tsgx11\", asset_id=\"111\", condition=\"false\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-traefik-crd-zttd2\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"e35afc00-e38e-48a0-ae9a-3eb25b74a58c\"}" timestamp=1689545119276
Jul 17 03:06:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:31.744Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:00-0000:00:01.1-0000:01:00.0-infiniband_mad-umad0.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689545122644
Jul 17 03:06:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:31.744Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_info\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_endpoint=\"traefik\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545119276
Jul 17 03:06:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:31.759Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"61\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545120791
Jul 17 03:06:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:31.765Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"117\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545120791
Jul 17 03:06:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:31.782Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"28\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545120791
Jul 17 03:06:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:31.828Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_aof_enabled\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545116711
Jul 17 03:06:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:31.832Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dracut-pre-mount.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689545126907
Jul 17 03:06:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:31.844Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"115\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545120791
Jul 17 03:06:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:31.847Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-ens6f0np0.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689545122644
Jul 17 03:06:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:31.854Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_annotations\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_endpoint=\"cm-policy-local-cache-redis-headless\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545119276
Jul 17 03:06:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:31.862Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_address_available\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_endpoint=\"shaping-announce-port-tsg-traffic-engine-numa-0\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545119276
Jul 17 03:06:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:31.868Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"usr_rx_drop_total\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", dev=\"ens7f1np1\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"mrzcpd-exporter\"}" timestamp=1689545119330
Jul 17 03:06:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:31.877Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"104\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545120791
Jul 17 03:06:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:31.948Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"49\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545120791
Jul 17 03:06:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:31.963Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"var-lib-kubelet-pods-b7e44624\\\\x2d9b20\\\\x2d4189\\\\x2d9ac3\\\\x2db8a15c1776a6-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2dmwm7p.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689545122644
Jul 17 03:06:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:31.966Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dracut-shutdown-onfailure.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"oneshot\", type=\"service\"}" timestamp=1689545126907
Jul 17 03:06:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:31.983Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-a2e57f37c45138813d9e198674124e639ecea8fa26120419cf7993b2664bf915.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"scope\"}" timestamp=1689545126907
Jul 17 03:06:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:31.983Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_compressed_total\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth9a152f90\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545120791
Jul 17 03:06:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:31.989Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_status_start_time\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", job_name=\"helm-install-traefik\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545119276
Jul 17 03:06:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:31.997Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_mtu_bytes\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens2f1np1\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545120939
Jul 17 03:06:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:31.997Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"local-fs.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"target\"}" timestamp=1689545122644
Jul 17 03:06:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:32.005Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db190\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545116711
Jul 17 03:06:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:32.043Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"overlay\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-bf08c70678550e8e0be395f7cf5ece571236bf2c50c9718cce0c76c6dcfe94eb-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689545122644
Jul 17 03:06:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:32.129Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db181\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545116711
Jul 17 03:06:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:32.223Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_reservation_limit_bytes\", asset=\"msh-tsgx05\", asset_id=\"67\", container=\"prometheus-node-exporter\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545115836
Jul 17 03:06:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:32.239Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", phase=\"Pending\", pod=\"prometheus-server-6ff784d758-rmvjl\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"0fa5c602-236e-4657-a253-bfddabdcc9f1\"}" timestamp=1689545124068
Jul 17 03:06:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:32.251Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"119\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545120939
Jul 17 03:06:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:32.324Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_fully_labeled_replicas\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"dashboard-metrics-scraper-5b94897dc9\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545130436
Jul 17 03:06:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:32.341Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"nis-domainname.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689545126907
Jul 17 03:06:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:32.354Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"var.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689545122644
Jul 17 03:06:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:32.371Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_last_terminated_reason\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"lb-tcp-443\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"svclb-traefik-46c02faf-rzf44\", project=\"TSG-OS\", project_id=\"7\", reason=\"Unknown\", service=\"prometheus-kube-state-metrics\", uid=\"59e0f471-ccc6-4ff5-8571-7fb136f39b65\"}" timestamp=1689545119276
Jul 17 03:06:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:32.384Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"42\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545120791
Jul 17 03:06:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:32.452Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"85\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545120791
Jul 17 03:06:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:32.461Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_net_dev_group\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"eno36np3\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545120605
Jul 17 03:06:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:32.465Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"twa-tsgx05\", asset_id=\"99\", container=\"prometheus-pushgateway\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"sleeping\"}" timestamp=1689545115860
Jul 17 03:06:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:32.470Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db105\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545116711
Jul 17 03:06:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:32.477Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_mspan_inuse_bytes\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545120939
Jul 17 03:06:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:32.489Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db186\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545116711
Jul 17 03:06:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:32.495Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_info\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_endpoint=\"cm-policy-local-cache-redis-master\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545119894
Jul 17 03:06:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:32.536Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda5\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", fstype=\"ext4\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/var/lib/kubelet/pods/506c9da6-9ad1-4f6d-afc6-e4606512a111/volume-subpaths/config-volume/telegraf/1\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545120791
Jul 17 03:06:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:32.538Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_annotations\", asset=\"twa-tsgx05\", asset_id=\"99\", configmap=\"multus-cni-config\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545119894
Jul 17 03:06:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:32.541Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"twa-tsgx05\", asset_id=\"99\", condition=\"false\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", pod=\"tsg-traffic-engine-numa-1-firewall-6dd49d747-slxl4\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"bdd10b0d-cc33-4a4f-9622-89ebf267e6c3\"}" timestamp=1689545119894
Jul 17 03:06:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:32.556Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-2096ff4a\\\\x2d92d2\\\\x2d47a7\\\\x2d9b70\\\\x2d293a6568cee4-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2d8tn7x.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689545122644
Jul 17 03:06:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:32.571Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_flags\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens8f0np0\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545120605
Jul 17 03:06:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:32.602Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_fifo_total\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens4f2np2\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545120605
Jul 17 03:06:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:32.615Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-virtual-net-vethb98a16e3.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689545127691
Jul 17 03:06:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:32.860Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_metadata_generation\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"systemd-exporter\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545130436
Jul 17 03:06:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:32.920Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_packets_dropped_total\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"cadvisor\", interface=\"ens1f0np0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545119516
Jul 17 03:06:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:32.938Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth09efe48d\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545119093
Jul 17 03:06:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:32.954Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_state_started\", asset=\"twa-tsgx05\", asset_id=\"99\", container=\"helm\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-configmap-reloader-csz4v\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"f2a551b0-9b53-4e7a-a0d8-da304384770b\"}" timestamp=1689545119894
Jul 17 03:06:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:32.960Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"39\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545119093
Jul 17 03:06:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:33.041Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_terminated\", asset=\"twa-tsgx03\", asset_id=\"95\", container=\"dign-client\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"dign-client-nvg4d\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"6e410083-6316-4234-8b1e-7c79ee4588d4\"}" timestamp=1689545130436
Jul 17 03:06:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:33.058Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_annotations\", asset=\"msh-tsgx11\", asset_id=\"111\", configmap=\"coredns\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545119276
Jul 17 03:06:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:33.059Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"84\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545119093
Jul 17 03:06:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:34.205Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_queue_length\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth6de2e73d\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545120605
Jul 17 03:06:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:34.295Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_queue_length\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethc8144301\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545120605
Jul 17 03:06:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:34.319Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Udp_Link_C2S_ALL\", app_name=\"sapp4\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", vsysId=\"1\"}" timestamp=1689545123815
Jul 17 03:06:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:34.340Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-rfkill.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"service\"}" timestamp=1689545127691
Jul 17 03:06:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:34.446Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_rss\", asset=\"twa-tsgx05\", asset_id=\"99\", container=\"dign-client\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545103344
Jul 17 03:06:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:40.482Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db217\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545133827
Jul 17 03:06:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:40.520Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"59\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545131647
Jul 17 03:06:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:40.554Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-mrzcpd-exporter-xgz49\", project=\"TSG-OS\", project_id=\"7\", reason=\"Evicted\", service=\"prometheus-kube-state-metrics\", uid=\"301784b5-1927-45d7-84ae-687804ad7a5e\"}" timestamp=1689545140034
Jul 17 03:06:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:40.574Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-47a07edd11ecc492029afdaa1b6e5cbd6e3a32d5f3c9d5fa234e8487313888d9.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"scope\"}" timestamp=1689545135764
Jul 17 03:06:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:40.738Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_hbase_table_meta_metric_flushMemstoreSize_95th_percentile\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545139757
Jul 17 03:06:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:40.760Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"loadmodules.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"service\"}" timestamp=1689545135764
Jul 17 03:06:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:40.769Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_compactionsQueuedCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"session_record_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"1532e1f3abf26580a1df4dec47d68996\", table=\"index_partfile_session_record_hos_bucket\", type=\"index\"}" timestamp=1689545139757
Jul 17 03:06:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:40.800Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_waiting\", asset=\"twa-tsgx04\", asset_id=\"97\", container=\"kube-multus\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"kube-multus-ds-k628t\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"53ae4357-99c5-46f0-b8e1-ea82fe17f5b4\"}" timestamp=1689545140034
Jul 17 03:06:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:40.821Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-prometheus-vzwtx\", project=\"TSG-OS\", project_id=\"7\", reason=\"UnexpectedAdmissionError\", service=\"prometheus-kube-state-metrics\", uid=\"1187ac6c-e3c7-4185-8a91-8bb4228b3b51\"}" timestamp=1689545140034
Jul 17 03:06:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:40.863Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"twa-tsgx04\", asset_id=\"97\", container=\"redis\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"cadvisor\", failure_type=\"pgfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"container\"}" timestamp=1689545115208
Jul 17 03:06:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:40.892Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"123\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545131647
Jul 17 03:06:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:40.903Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failcnt\", asset=\"twa-tsgx04\", asset_id=\"97\", container=\"local-path-provisioner\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545114155
Jul 17 03:06:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:40.969Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"twa-tsgx04\", asset_id=\"97\", container=\"telegraf-security\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", state=\"stopped\"}" timestamp=1689545110944
Jul 17 03:06:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:40.986Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_waiting\", asset=\"twa-tsgx04\", asset_id=\"97\", container=\"prometheus-server-configmap-reload\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-server-6ff784d758-bfmc9\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"497bfad5-323f-4476-8405-f97cffdaa52c\"}" timestamp=1689545140034
Jul 17 03:06:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:41.026Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_maxFlushQueueSize\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"proxy_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"29196d643831de92c5633f334a8ada62\", table=\"index_partfile_proxy_hos_bucket\", type=\"index\"}" timestamp=1689545139757
Jul 17 03:06:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:41.045Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_alloc_bytes_total\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545133827
Jul 17 03:06:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:41.062Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_frame_total\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"eno35np2\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545131647
Jul 17 03:06:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:41.081Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_10_193_1_metric_majorCompactionInputFileCount_99th_percentile\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545139757
Jul 17 03:06:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:41.173Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_gc_duration_seconds_sum\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545133827
Jul 17 03:06:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:41.187Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_errs_total\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethbcbc9544\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545131647
Jul 17 03:06:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:41.199Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"52\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545131647
Jul 17 03:06:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:41.283Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx04\", asset_id=\"97\", container=\"firewall\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Total\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689545112585
Jul 17 03:06:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:41.336Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_cpu_shares\", asset=\"twa-tsgx04\", asset_id=\"97\", container=\"mrapm-device\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545128797
Jul 17 03:06:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:41.337Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_avgStoreFileAge\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"6d8434ebf590828f20e12182bb817aa5\", table=\"rtp_hos_bucket\", type=\"data\"}" timestamp=1689545139757
Jul 17 03:06:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:41.342Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dracut-pre-mount.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"service\"}" timestamp=1689545135764
Jul 17 03:06:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:41.343Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"cadvisor\", failure_type=\"pgmajfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"hierarchy\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689545127828
Jul 17 03:06:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:41.352Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_restart_policy\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"metrics-server-54dc485875-lkr7j\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", type=\"Always\", uid=\"4f4807f1-9973-4c05-9357-9438c02bd87b\"}" timestamp=1689545140034
Jul 17 03:06:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:41.356Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_LOG_metric_putTime_max\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545139757
Jul 17 03:06:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:41.361Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_info\", asset=\"twa-tsgx04\", asset_id=\"97\", container=\"helm\", container_id=\"containerd://693c6841179ead67aab2814dc04b0bfef8556f8d65202acee9b3540311f1d5f0\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", image=\"docker.io/rancher/klipper-helm:v0.8.0-build20230510\", image_id=\"sha256:6f42df210d7faa0b6eb56f5f28bb061930c7834af1e7dd4b3586108ec6b1485b\", image_spec=\"rancher/klipper-helm:v0.8.0-build20230510\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-cm-policy-local-cache-rxfmw\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"9e8ca26d-6038-4672-99ec-32891ca961a5\"}" timestamp=1689545140034
Jul 17 03:06:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:41.361Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-mrzcpd-huge_pages.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689545135764
Jul 17 03:06:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:41.379Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"92\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545131647
Jul 17 03:06:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:41.420Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"112\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545131647
Jul 17 03:06:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:41.427Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db118\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545133827
Jul 17 03:06:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:41.496Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_carrier_total\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"eno33np0\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545131647
Jul 17 03:06:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:41.545Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db255\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545133827
Jul 17 03:06:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:41.554Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_threads_max\", asset=\"twa-tsgx04\", asset_id=\"97\", container=\"lb-tcp-80\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545106535
Jul 17 03:06:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:41.578Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_netstat_Udp6_NoPorts\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545131647
Jul 17 03:06:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:41.580Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-a5104a71fcffa4fdb02c774d48755d39d7fbc70c216860da9d967bb75977a92f.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"scope\"}" timestamp=1689545135764
Jul 17 03:06:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:41.581Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Pending\", pod=\"helm-install-configmap-reloader-dtgld\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"81ee341a-8021-4c5c-8cff-614ea7d231b5\"}" timestamp=1689545140034
Jul 17 03:06:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:41.619Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_flags\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens4f1np1\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545131647
Jul 17 03:06:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:41.632Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_type\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"kubernetes-dashboard-certs\", service=\"prometheus-kube-state-metrics\", type=\"Opaque\"}" timestamp=1689545140034
Jul 17 03:06:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:41.635Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Tcp_Offload_Bit\", app_name=\"sapp4\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", vsysId=\"1\"}" timestamp=1689545139161
Jul 17 03:06:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:41.655Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_numReferenceFiles\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"firewall_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"bcee559236e0be70ad0a1e9d2a915a48\", table=\"index_partfile_firewall_hos_bucket\", type=\"index\"}" timestamp=1689545139757
Jul 17 03:06:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:41.756Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", fstype=\"ext4\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/data/var/lib/kubelet/pods/9ad11045-e0bd-4481-9c2d-4c81a67475ee/volume-subpaths/config-volume/telegraf-security/2\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545131647
Jul 17 03:06:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:42.077Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_lastMajorCompactionAge\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", namespace=\"SYSTEM\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"4452b33c145b028ad09199ea5390d4ae\", table=\"LOG\", type=\"data\"}" timestamp=1689545139757
Jul 17 03:06:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:42.083Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_getCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"proxy_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"c48ff79d9c4166508d5a5bc3b3ba98c5\", table=\"index_partfile_proxy_hos_bucket\", type=\"index\"}" timestamp=1689545139757
Jul 17 03:06:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:42.094Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_lastMajorCompactionAge\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"default\", project=\"OLAP\", project_id=\"3\", region=\"86f7102593856c52aa071d1a2a7fdc4e\", table=\"index_partfile_default\", type=\"index\"}" timestamp=1689545139757
Jul 17 03:06:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:42.245Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx06\", asset_id=\"61\", container=\"exporter-merger\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx06\", endpoint_id=\"429\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"1\", operation=\"Async\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545118794
Jul 17 03:06:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:42.258Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_compressed_total\", asset=\"msh-tsgx06\", asset_id=\"61\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"eno33np0\", endpoint=\"TSG-OS-msh-tsgx06\", endpoint_id=\"429\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545127064
Jul 17 03:06:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:42.481Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_packets_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"cadvisor\", interface=\"ens3f1np1\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545139540
Jul 17 03:06:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:42.627Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-traefik-crd-r4msj\", project=\"TSG-OS\", project_id=\"7\", reason=\"NodeLost\", service=\"prometheus-kube-state-metrics\", uid=\"b4b0df3d-0f85-4950-9e3b-a5a66a88bc21\"}" timestamp=1689545135154
Jul 17 03:06:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:42.640Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", cpu=\"125\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545136242
Jul 17 03:06:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:42.659Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_timex_offset_seconds\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545136242
Jul 17 03:06:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:42.724Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", container=\"dign-client\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Discard\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545128403
Jul 17 03:06:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:42.736Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_load_average_10s\", asset=\"pcap-tsgx01\", asset_id=\"71\", container=\"telegraf-security\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689545133340
Jul 17 03:06:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:42.744Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_max_usage_bytes\", asset=\"pcap-tsgx01\", asset_id=\"71\", container=\"lb-tcp-80\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545134635
Jul 17 03:06:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:42.755Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-configmap-reloader-24srf\", project=\"TSG-OS\", project_id=\"7\", reason=\"UnexpectedAdmissionError\", service=\"prometheus-kube-state-metrics\", uid=\"f19bfeb9-dc33-4eae-b23d-70b3714ba69c\"}" timestamp=1689545135154
Jul 17 03:06:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:42.979Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"mysql_global_variables_binlog_commit_wait_count\", asset=\"msh-dellr740-olap002\", asset_id=\"13\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Mariadb-msh-dellr740-olap002\", endpoint_id=\"69\", module=\"NC-Mariadb\", module_id=\"43\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545142786
Jul 17 03:06:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:43.652Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_info\", asset=\"pcap-tsgx01\", asset_id=\"71\", container=\"telegraf\", container_id=\"containerd://5611dc73bcd87d48480acece3af195cc66b4bca02907f9bb4b42e627b59bd1c9\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", image=\"registry.gdnt-cloud.website/tsg-telegraf:v23.07-rc1-079e964\", image_id=\"sha256:2cebbbb83cda415792ef0a3a045c352ff7e754b37cf4f9b9002be26297e1306f\", image_spec=\"registry.gdnt-cloud.website/tsg-telegraf:v23.07-rc1-079e964\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", pod=\"tsg-traffic-engine-numa-1-firewall-69ff8c6fd4-fzrff\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"a38bc0a0-f54d-4548-b0f1-6472f0282ca8\"}" timestamp=1689545135154
Jul 17 03:06:44 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:44.012Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_flags\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"eno34np1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545136242
Jul 17 03:06:44 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:44.394Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"mysql_global_status_commands_total\", asset=\"msh-dellr740-olap002\", asset_id=\"13\", command=\"show_charsets\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Mariadb-msh-dellr740-olap002\", endpoint_id=\"69\", module=\"NC-Mariadb\", module_id=\"43\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545142786
Jul 17 03:06:44 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:44.660Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"mysql_global_status_commands_total\", asset=\"msh-dellr740-olap002\", asset_id=\"13\", command=\"ha_read\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Mariadb-msh-dellr740-olap002\", endpoint_id=\"69\", module=\"NC-Mariadb\", module_id=\"43\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545142786
Jul 17 03:06:48 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:48.637Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_compactionsQueuedCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"firewall_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"e4331eab37a02574cfac02404cf8f4bf\", table=\"index_filename_firewall_hos_bucket\", type=\"index\"}" timestamp=1689545147443
Jul 17 03:06:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:49.223Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_default_metric_scanTime_max\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545147443
Jul 17 03:06:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:49.238Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_3_metric_splitTime_num_ops\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545147443
Jul 17 03:06:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:49.496Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_constraint_errors_transmitted_total\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_0\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545141797
Jul 17 03:06:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:49.661Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_address_assign_type\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens4f2np2\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545141797
Jul 17 03:06:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:49.661Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_flushesQueuedCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"firewall_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"569c4ca2944511d5d885d0d8a8bc9016\", table=\"index_partfile_firewall_hos_bucket\", type=\"index\"}" timestamp=1689545147443
Jul 17 03:06:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:49.686Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_PauseTimeWithoutGc_99_9th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Server\"}" timestamp=1689545147443
Jul 17 03:06:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:49.742Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_fifo_total\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth2ecac7c4\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545141797
Jul 17 03:06:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:49.746Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_session_record_hos_bucket_metric_deleteBatchTime_98th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545147443
Jul 17 03:06:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:49.770Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_session_record_hos_bucket_metric_flushMemstoreSize_mean\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545147443
Jul 17 03:06:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:49.791Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_193_1_metric_putTime_max\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545147443
Jul 17 03:06:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:49.896Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_3_metric_majorCompactedInputBytes\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545147443
Jul 17 03:06:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:49.920Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_hwmon_temp_celsius\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", chip=\"platform_coretemp_0\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-olap003\", endpoint_id=\"235\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"yarn_resourcemanager,zookeeper,nacos,kafka,spark,druid_data,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\", sensor=\"temp1\"}" timestamp=1689545149713
Jul 17 03:06:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:49.997Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_session_record_hos_bucket_metric_compactionOutputSize_max\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545147443
Jul 17 03:06:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:50.015Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"60\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545146709
Jul 17 03:06:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:50.026Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_incrementCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"firewall_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"efdc1c66d4c62e1cb4d0bf16a4f8d36c\", table=\"index_partfile_firewall_hos_bucket\", type=\"index\"}" timestamp=1689545147443
Jul 17 03:06:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:50.070Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_boot_time_seconds\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-olap003\", endpoint_id=\"235\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"yarn_resourcemanager,zookeeper,nacos,kafka,spark,druid_data,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545149713
Jul 17 03:06:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:50.126Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_reads_merged_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"overlay_0-95\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545144170
Jul 17 03:06:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:50.146Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"twa-tsgx08\", asset_id=\"105\", container=\"dign-server-web\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", failure_type=\"pgfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"hierarchy\"}" timestamp=1689545144021
Jul 17 03:06:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:50.184Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_terminated_reason\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"helm\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-traefik-cmxhd\", project=\"TSG-OS\", project_id=\"7\", reason=\"Completed\", service=\"prometheus-kube-state-metrics\", uid=\"d8416f0e-99a7-4879-b78a-552b098f6242\"}" timestamp=1689545139668
Jul 17 03:06:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:50.278Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_start_time_seconds\", asset=\"twa-tsgx08\", asset_id=\"105\", container=\"configmap-reloader\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545147561
Jul 17 03:06:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:50.339Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_spec_parallelism\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", job_name=\"helm-install-configmap-reloader\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545139668
Jul 17 03:06:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:50.482Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_local_link_integrity_errors_total\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_4\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545141797
Jul 17 03:06:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:50.564Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_scaling_frequency_hertz\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545146709
Jul 17 03:06:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:50.589Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_labels\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-node-exporter-bz7wf\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"d58b8827-5397-48fd-9ce6-3976a10dcb09\"}" timestamp=1689545139668
Jul 17 03:06:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:50.614Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx10\", asset_id=\"109\", cpu=\"72\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545141797
Jul 17 03:06:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:50.636Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_queue_length\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens2f0np0\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545141797
Jul 17 03:06:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:50.641Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_sector_reads_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/data/var/lib/kubelet/pods/3a7372b8-22f7-4934-bc38-fba12a319dfc/volumes/kubernetes.io~projected/kube-api-access-x5c6f\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545144170
Jul 17 03:06:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:50.713Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_threads_max\", asset=\"twa-tsgx08\", asset_id=\"105\", container=\"prometheus-server-configmap-reload\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545139802
Jul 17 03:06:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:50.755Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_scrape_collector_duration_seconds\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", collector=\"hwmon\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-olap003\", endpoint_id=\"235\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"yarn_resourcemanager,zookeeper,nacos,kafka,spark,druid_data,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545149713
Jul 17 03:06:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:50.854Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.grpc.v1.cri-sandboxes-0ae8a62036ecb47be4d373967cb7cf49e6728bc25282ace892d4034a6f967b5f-shm.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689545146554
Jul 17 03:06:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:50.996Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_usage_bytes\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"overlay_0-267\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545144170
Jul 17 03:06:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:51.130Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_fully_labeled_replicas\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-1-firewall-69d94d4d6b\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545139668
Jul 17 03:06:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:51.312Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_mtu_bytes\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"docker0\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-olap003\", endpoint_id=\"235\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"yarn_resourcemanager,zookeeper,nacos,kafka,spark,druid_data,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545149713
Jul 17 03:06:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:52.835Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_link_downed_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"mlx5_6\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545149337
Jul 17 03:06:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:53.042Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"msh-tsgx08\", asset_id=\"65\", container=\"prometheus-alertmanager-configmap-reload\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"uninterruptible\"}" timestamp=1689545130160
Jul 17 03:06:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:53.128Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_name_assign_type\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth7bb95ccb\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545149337
Jul 17 03:06:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:53.314Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-pod67eafe01_721d_41b1_93f6_903b88527edb.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"slice\"}" timestamp=1689545151179
Jul 17 03:06:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:53.350Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.grpc.v1.cri-sandboxes-eb97a177fd211b3bdbde038edf910d5a998032e32a5ad10349cc1b6fe624f848-shm.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689545151179
Jul 17 03:06:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:53.470Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Total\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545149658
Jul 17 03:06:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:53.557Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"pcap-tsgx11\", asset_id=\"115\", condition=\"Available\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"metrics-server\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"unknown\"}" timestamp=1689545144478
Jul 17 03:06:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:53.625Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_queue_length\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"eno1\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545148328
Jul 17 03:06:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:53.640Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_sockstat_TCP6_inuse\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545148328
Jul 17 03:06:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:53.834Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Udp_Link_S2C\", app_name=\"sapp4\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", vsysId=\"1\"}" timestamp=1689545142596
Jul 17 03:06:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:53.979Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_status_active\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", job_name=\"helm-delete-tsg-traffic-engine-numa-1\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545148950
Jul 17 03:06:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:53.985Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_tasks_max\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"initrd-parse-etc.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689545140884
Jul 17 03:06:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:54.028Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx08\", asset_id=\"65\", cpu=\"9\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545148328
Jul 17 03:06:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:54.060Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_labels\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.systemd-exporter.v1\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545144478
Jul 17 03:06:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:54.077Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_receive_remote_physical_errors_total\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_0\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545148328
Jul 17 03:06:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:54.081Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", cpu=\"11\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545149337
Jul 17 03:06:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:54.131Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"vethebc85272\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545148328
Jul 17 03:06:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:54.205Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_cpu_seconds_total\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-pod2c4890df_bfb7_48e8_a274_494cc0f3691d.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"slice\"}" timestamp=1689545152135
Jul 17 03:06:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:54.417Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"pcap-tsgx07\", asset_id=\"85\", condition=\"true\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"metrics-server-54dc485875-jkvd8\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"0a0a8b78-3fe6-4b35-93e0-3e1bdd94459c\"}" timestamp=1689545142058
Jul 17 03:06:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:54.446Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-9c8e5ab12d9bd98a49dbd59995a5a20f24191ed7c3c5ca8c7c712d14633fd74a-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689545140884
Jul 17 03:06:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:54.519Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"30\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545139409
Jul 17 03:06:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:54.538Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cpu=\"95\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545139409
Jul 17 03:06:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:54.567Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-cm-policy-local-cache-5ktrg\", project=\"TSG-OS\", project_id=\"7\", reason=\"UnexpectedAdmissionError\", service=\"prometheus-kube-state-metrics\", uid=\"6b46916c-eeef-4c1f-97c6-88043a8c1ae7\"}" timestamp=1689545148950
Jul 17 03:06:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:54.604Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", container=\"prometheus-pushgateway\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"cadvisor\", failure_type=\"pgmajfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"hierarchy\"}" timestamp=1689545143844
Jul 17 03:06:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:54.613Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", state=\"uninterruptible\"}" timestamp=1689545147666
Jul 17 03:06:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:54.649Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_frame_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"br_dign_c\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545139409
Jul 17 03:06:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:54.670Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_name_assign_type\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"bond0\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545139409
Jul 17 03:06:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:54.709Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_address_available\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_endpoint=\"kube-dns\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545144478
Jul 17 03:06:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:54.794Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", cpu=\"91\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545149337
Jul 17 03:06:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:54.829Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_labels\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", pod=\"dashboard-metrics-scraper-5b94897dc9-jk6qk\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"17060853-6cdd-48b9-836a-8dba7f276cf9\"}" timestamp=1689545142058
Jul 17 03:06:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:54.870Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_gc_sys_bytes\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545138659
Jul 17 03:06:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:54.880Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_max_usage_bytes\", asset=\"msh-tsgx08\", asset_id=\"65\", container=\"kube-state-metrics\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545137189
Jul 17 03:06:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:54.909Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas_ready\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"ipmi-exporter\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545148950
Jul 17 03:06:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:54.932Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_cpu_seconds_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"slice\"}" timestamp=1689545140884
Jul 17 03:06:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:54.936Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_errors_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"cadvisor\", interface=\"ens3f0np0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545142777
Jul 17 03:06:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:54.951Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_id\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"br_dign_c\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545143391
Jul 17 03:06:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:54.954Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_namespace_status_phase\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"consul\", nz_agent_id=\"3\", phase=\"Active\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545142058
Jul 17 03:06:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:54.982Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_files\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"shm\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/2cc6af6d20215f80e13f9fef6cc57238d803fcb324abd18e26ef9b40203bbb40/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545143391
Jul 17 03:06:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:55.003Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_changes_total\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"lo\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545148328
Jul 17 03:06:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:55.059Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_name_assign_type\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth8fe690c2\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545148328
Jul 17 03:06:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:55.060Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-67e13d8948a718c4bb0914ee7b1365dd1ea297468f31a3c98635ade831443f13.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"scope\"}" timestamp=1689545140884
Jul 17 03:06:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:55.080Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cpu=\"118\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545139409
Jul 17 03:06:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:55.193Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"6\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545149337
Jul 17 03:06:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:55.244Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_info\", asset=\"pcap-tsgx07\", asset_id=\"85\", configmap=\"chart-content-traefik\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545142058
Jul 17 03:06:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:55.274Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_flags\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"eno33np0\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545139409
Jul 17 03:06:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:55.275Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_exporter_last_scrape_error\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545139367
Jul 17 03:06:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:55.469Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", cpu=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545149337
Jul 17 03:06:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:55.988Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_last_seen\", asset=\"pcap-tsgx07\", asset_id=\"85\", container=\"metrics-server\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545150626
Jul 17 03:06:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:56.022Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"rpc-gssd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"service\"}" timestamp=1689545140884
Jul 17 03:06:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:56.028Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_start_time_seconds\", asset=\"msh-tsgx08\", asset_id=\"65\", container=\"firewall\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689545137907
Jul 17 03:06:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:56.063Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx08\", asset_id=\"65\", cpu=\"92\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545148328
Jul 17 03:06:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:56.067Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_threads_max\", asset=\"pcap-tsgx07\", asset_id=\"85\", container=\"dashboard-metrics-scraper\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545135624
Jul 17 03:06:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:56.081Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_observed_generation\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"coredns\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545142058
Jul 17 03:06:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:56.111Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx12\", asset_id=\"113\", cpu=\"46\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545143391
Jul 17 03:06:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:56.138Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db24\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545138659
Jul 17 03:06:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:56.157Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"msh-tsgx08\", asset_id=\"65\", condition=\"Progressing\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"tsg-traffic-engine-numa-1-firewall\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"unknown\"}" timestamp=1689545148950
Jul 17 03:06:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:56.200Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"msh-tsgx08\", asset_id=\"65\", container=\"ipmi-exporter\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"uninterruptible\"}" timestamp=1689545128643
Jul 17 03:06:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:56.246Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_free_bytes\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"shm\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/2cc6af6d20215f80e13f9fef6cc57238d803fcb324abd18e26ef9b40203bbb40/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545143391
Jul 17 03:06:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:56.271Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_multicast_packets_transmitted_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"mlx5_4\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545145883
Jul 17 03:06:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:56.349Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"msh-tsgx12\", asset_id=\"113\", cpu=\"98\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545143391
Jul 17 03:06:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:56.701Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_cpu_seconds_total\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-burstable-pod8b2e9710_4fd8_4c6f_8e5f_798d330e80a6.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"slice\"}" timestamp=1689545151179
Jul 17 03:06:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:57.244Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_info\", asset=\"pcap-tsgx03\", asset_id=\"73\", board_id=\"MT_0000000359\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"mlx5_1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", firmware_version=\"22.28.4000\", hca_type=\"MT4125\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545145883
Jul 17 03:06:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:57.410Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"pcap-tsgx03\", asset_id=\"73\", condition=\"Available\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"systemd-exporter\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"true\"}" timestamp=1689545153389
Jul 17 03:06:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:57.514Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled_time\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"local-path-provisioner-957fdf8bc-qmcnx\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"43749d42-dc70-4f09-84cc-b9a556f85f71\"}" timestamp=1689545153389
Jul 17 03:06:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:57.711Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_created\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", job_name=\"helm-install-systemd-exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545153389
Jul 17 03:06:58 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:58.162Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_annotations\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_endpoint=\"prometheus-node-exporter\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545153389
Jul 17 03:06:58 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:58.196Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_daemonset_status_number_unavailable\", asset=\"pcap-tsgx03\", asset_id=\"73\", daemonset=\"prometheus-node-exporter\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545153389
Jul 17 03:06:58 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:58.209Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_free_bytes\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"shm\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/d83b032fde63030c5f8fe3bc0bf131ea5ab051a044076812ffbb866b5a01c638/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545145883
Jul 17 03:06:58 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:58.225Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_dormant\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens6f1np1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545145883
Jul 17 03:06:58 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:58.308Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-dellr740-dt003\", asset_id=\"9\", cpu=\"29\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-dt003\", endpoint_id=\"229\", mode=\"system\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"hbase_master,clickhouse_data,kafka,gohangout,zookeeper,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545157407
Jul 17 03:06:58 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:58.352Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", container=\"kubernetes-dashboard\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"iowaiting\"}" timestamp=1689545148926
Jul 17 03:06:58 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:58.632Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"15\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545145883
Jul 17 03:06:58 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:58.822Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-aebe8d09\\\\x2d6d5d\\\\x2d4eda\\\\x2db90d\\\\x2d7be24e2da709-volume\\\\x2dsubpaths-config\\\\x2dvolume-telegraf\\\\x2dsecurity-1.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689545143372
Jul 17 03:06:58 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:58.892Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"58\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545145883
Jul 17 03:06:58 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:58.964Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", container=\"redis\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"running\"}" timestamp=1689545138984
Jul 17 03:06:58 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:58.981Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_replicas\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"cm-policy-local-cache-redis-master-69967b5879\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545153389
Jul 17 03:06:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:59.074Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"39\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545145883
Jul 17 03:06:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:59.701Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"39\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545149083
Jul 17 03:06:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:59.828Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_fifo_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"eno34np1\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545149083
Jul 17 03:06:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:59.852Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_compressed_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"cnibr0\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545149083
Jul 17 03:06:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:06:59.985Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_mcache_sys_bytes\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"OLAP-Node-Exporter-pcap-dellr740-dt001\", endpoint_id=\"251\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545157557
Jul 17 03:07:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:00.012Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_scaling_frequency_hertz\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", cpu=\"20\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"OLAP-Node-Exporter-pcap-dellr740-dt001\", endpoint_id=\"251\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545157557
Jul 17 03:07:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:00.040Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", cpu=\"39\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"OLAP-Node-Exporter-pcap-dellr740-dt001\", endpoint_id=\"251\", mode=\"steal\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545157557
Jul 17 03:07:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:00.069Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link_mode\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"em2\", endpoint=\"OLAP-Node-Exporter-pcap-dellr740-dt001\", endpoint_id=\"251\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545157557
Jul 17 03:07:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:00.419Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"64\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545149083
Jul 17 03:07:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:00.436Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_fifo_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"br_dign_s\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545149083
Jul 17 03:07:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:00.572Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"nfs-client.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"target\"}" timestamp=1689545144879
Jul 17 03:07:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:00.666Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_gc_sys_bytes\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545145071
Jul 17 03:07:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:00.687Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_last_terminated_reason\", asset=\"msh-tsgx01\", asset_id=\"51\", container=\"firewall\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", pod=\"tsg-traffic-engine-numa-0-firewall-57597b866c-6hbxk\", project=\"TSG-OS\", project_id=\"7\", reason=\"Error\", service=\"prometheus-kube-state-metrics\", uid=\"55b23504-098e-4605-9a10-91ab01c5f1f9\"}" timestamp=1689545147425
Jul 17 03:07:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:00.714Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"95\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545149083
Jul 17 03:07:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:00.903Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx01\", asset_id=\"51\", container=\"prometheus-pushgateway\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Discard\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545139987
Jul 17 03:07:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:01.052Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_labels\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", job_name=\"helm-delete-tsg-traffic-engine-numa-0\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545147425
Jul 17 03:07:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:02.867Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_scrape_error\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545152003
Jul 17 03:07:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:03.179Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_compressed_total\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth1977a0c\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-dt001\", endpoint_id=\"225\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545162988
Jul 17 03:07:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:03.237Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx02\", asset_id=\"63\", container=\"dign-client\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Discard\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545136252
Jul 17 03:07:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:03.249Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"prometheus_http_request_duration_seconds_bucket\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", handler=\"/api/v1/metadata\", instance=\"10.10.20.159:10090\", le=\"0.4\", type=\"global\"}" timestamp=1689545162990
Jul 17 03:07:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:03.337Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_usage_bytes\", asset=\"msh-tsgx02\", asset_id=\"63\", container=\"prometheus-node-exporter\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545148697
Jul 17 03:07:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:03.361Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_namespace_annotations\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-node-lease\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545151412
Jul 17 03:07:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:03.558Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_exporter_build_info\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", branch=\"HEAD\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-dt001\", endpoint_id=\"225\", goversion=\"go1.16.7\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\", revision=\"26645363b486e12be40af7ce4fc91e731a33104e\", version=\"1.2.2\"}" timestamp=1689545162988
Jul 17 03:07:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:03.673Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-dt001\", endpoint_id=\"225\", module=\"OLAP-Node-Exporter\", module_id=\"51\", name=\"2\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\", type=\"Processor\"}" timestamp=1689545162988
Jul 17 03:07:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:03.706Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"net_conntrack_dialer_conn_closed_total\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", dialer_name=\"asset_ping_65\", instance=\"10.10.20.159:10090\", type=\"global\"}" timestamp=1689545162990
Jul 17 03:07:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:03.723Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"net_conntrack_dialer_conn_failed_total\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", dialer_name=\"asset_ping_19\", instance=\"10.10.20.159:10090\", reason=\"timeout\", type=\"global\"}" timestamp=1689545162990
Jul 17 03:07:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:03.729Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"prometheus_http_request_duration_seconds_bucket\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", handler=\"/api/v1/targets\", instance=\"10.10.20.159:10090\", le=\"8\", type=\"global\"}" timestamp=1689545162990
Jul 17 03:07:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:03.739Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"net_conntrack_dialer_conn_failed_total\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", dialer_name=\"asset_ping_43\", instance=\"10.10.20.159:10090\", reason=\"refused\", type=\"global\"}" timestamp=1689545162990
Jul 17 03:07:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:03.776Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"prometheus_http_response_size_bytes_bucket\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", handler=\"/api/v1/label/:name/values\", instance=\"10.10.20.159:10090\", le=\"1000\", type=\"global\"}" timestamp=1689545162990
Jul 17 03:07:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:03.777Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_gc_duration_seconds\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", quantile=\"0.25\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545149269
Jul 17 03:07:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:03.832Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_free_bytes\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"tmpfs\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-dt001\", endpoint_id=\"225\", fstype=\"tmpfs\", module=\"OLAP-Node-Exporter\", module_id=\"51\", mountpoint=\"/run\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545162988
Jul 17 03:07:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:03.854Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", cpu=\"30\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-dt001\", endpoint_id=\"225\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545162988
Jul 17 03:07:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:03.873Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"net_conntrack_dialer_conn_attempted_total\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", dialer_name=\"asset_ping_3\", instance=\"10.10.20.159:10090\", type=\"global\"}" timestamp=1689545162990
Jul 17 03:07:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:03.886Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_fifo_total\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth5d6aa4a\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-dt001\", endpoint_id=\"225\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545162988
Jul 17 03:07:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:03.911Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", cpu=\"6\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-dt001\", endpoint_id=\"225\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545162988
Jul 17 03:07:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:03.983Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"0\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545158318
Jul 17 03:07:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:03.984Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"msh-tsgx02\", asset_id=\"63\", condition=\"unknown\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-alertmanager-6df7f457b6-tk42l\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"b4a34040-e633-4749-bfb3-b03089d307fe\"}" timestamp=1689545151412
Jul 17 03:07:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:04.002Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"msh-tsgx02\", asset_id=\"63\", cpu=\"4\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545158318
Jul 17 03:07:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:04.025Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"tmpfs\", name=\"run-k3s-containerd-io.containerd.grpc.v1.cri-sandboxes-ddbcb714f06b29a74c84e7a2f7db4c11054ee54ac9cac409f3130545064c6795-shm.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689545148447
Jul 17 03:07:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:04.049Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_annotations\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"chart-values-configmap-reloader\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545151412
Jul 17 03:07:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:04.068Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_tasks_max\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dracut-pre-udev.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689545148447
Jul 17 03:07:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:04.069Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"64\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545158318
Jul 17 03:07:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:04.160Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_changes_total\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth2ff062e3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545158318
Jul 17 03:07:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:04.210Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"tsg_master_del\", app_name=\"tsg_master\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"traffic-engine\", line_name=\"compile\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", table_name=\"sync_exdata\", vsysId=\"1\"}" timestamp=1689545153892
Jul 17 03:07:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:06.619Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"emergency.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689545153582
Jul 17 03:07:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:06.806Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_annotations\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"mrzcpd-exporter-5cf7f967cc\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545161227
Jul 17 03:07:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:06.821Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cpu=\"102\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545163910
Jul 17 03:07:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:06.848Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cpu=\"83\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545163910
Jul 17 03:07:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:06.860Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_bytes_total\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"cadvisor\", interface=\"ens7f0np0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545157680
Jul 17 03:07:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:06.877Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"twa-tsgx01\", asset_id=\"91\", container=\"exporter\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"iowaiting\"}" timestamp=1689545151511
Jul 17 03:07:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:07.267Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_fifo_total\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens7f1np1\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545163910
Jul 17 03:07:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:07.285Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_limit_bytes\", asset=\"twa-tsgx01\", asset_id=\"91\", container=\"dign-server-web\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545162966
Jul 17 03:07:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:07.309Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cpu=\"41\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545163910
Jul 17 03:07:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:07.564Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_cpu_period\", asset=\"twa-tsgx01\", asset_id=\"91\", container=\"prometheus-server-configmap-reload\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545162966
Jul 17 03:07:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:07.695Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_cpu_period\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689545162966
Jul 17 03:07:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:08.052Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cpu=\"42\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545163910
Jul 17 03:07:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:08.465Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"cortex_blocks_meta_sync_duration_seconds_bucket\", agent_name=\"msh-kvm-nz001\", application=\"cortex\", component=\"querier\", datacenter=\"MSH\", instance=\"127.0.0.1:10093\", le=\"+Inf\"}" timestamp=1689545167261
Jul 17 03:07:09 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:09.632Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_deleteCount\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"firewall_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"ebb46fb579a2b3ca1735fe995db38a0a\", table=\"index_filename_firewall_hos_bucket\", type=\"index\"}" timestamp=1689545168804
Jul 17 03:07:09 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:09.972Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_lastMajorCompactionAge\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"rtp_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"d64e085bec7dcaecff2f6350cdf00ff6\", table=\"index_time_rtp_hos_bucket\", type=\"index\"}" timestamp=1689545168804
Jul 17 03:07:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:10.192Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_readRequestCount\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"firewall_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"700a07cb70ff9f2c61808c3e4df6599d\", table=\"index_filename_firewall_hos_bucket\", type=\"index\"}" timestamp=1689545168804
Jul 17 03:07:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:10.360Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_STATS_metric_majorCompactionTime_90th_percentile\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545168804
Jul 17 03:07:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:10.913Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_readRequestCount\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"default\", project=\"OLAP\", project_id=\"3\", region=\"8fdbb0afc100cef71526519ab87bf161\", table=\"index_partfile_default\", type=\"index\"}" timestamp=1689545168804
Jul 17 03:07:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:11.625Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_getCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"49caac8a08a11079694f8ae38226a71f\", table=\"session_record_hos_bucket\", type=\"data\"}" timestamp=1689545170147
Jul 17 03:07:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:11.706Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_194_3_metric_scanSize_min\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545170147
Jul 17 03:07:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:11.801Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_194_4_metric_majorCompactionOutputFileCount_99_9th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545170147
Jul 17 03:07:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:11.871Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_4_metric_deleteTime_99_9th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545170147
Jul 17 03:07:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:11.957Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_session_record_hos_bucket_metric_deleteTime_max\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545170147
Jul 17 03:07:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:11.987Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_session_record_hos_bucket_metric_splitTime_min\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545170147
Jul 17 03:07:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:12.493Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_deleteCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"default\", project=\"OLAP\", project_id=\"3\", region=\"08440c759997947c879bb5389a3e7c5e\", table=\"index_partfile_default\", type=\"index\"}" timestamp=1689545170147
Jul 17 03:07:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:12.523Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_session_record_hos_bucket_metric_majorCompactionOutputFileCount_99th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545170147
Jul 17 03:07:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:12.537Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_scanCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"7b846d792b421ff5ea5dc9cbcef14f50\", table=\"firewall_hos_bucket\", type=\"data\"}" timestamp=1689545170147
Jul 17 03:07:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:12.620Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_tsg_galaxy_table_relation_framedip_account_metric_deleteTime_95th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545170147
Jul 17 03:07:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:12.670Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_FsReadTime_min\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"IO\"}" timestamp=1689545170147
Jul 17 03:07:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:12.802Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Delete_90th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Server\"}" timestamp=1689545170147
Jul 17 03:07:13 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:13.258Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_tsg_galaxy_table_relation_framedip_account_metric_getTime_99_9th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545170147
Jul 17 03:07:13 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:13.282Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_session_record_hos_bucket_metric_compactionInputFileCount_95th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545170147
Jul 17 03:07:13 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:13.300Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_alloc_bytes\", asset=\"twa-dellr740-dt002\", asset_id=\"39\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"NC-Gohangout-twa-dellr740-dt002\", endpoint_id=\"179\", module=\"DC-Gohangout\", module_id=\"35\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545171532
Jul 17 03:07:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:15.474Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_changes_total\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens3f0np0\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545161230
Jul 17 03:07:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:15.647Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_cpu_shares\", asset=\"twa-tsgx07\", asset_id=\"103\", container=\"kube-state-metrics\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545169150
Jul 17 03:07:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:15.800Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_fully_labeled_replicas\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"coredns-65dfb4fcf4\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545168076
Jul 17 03:07:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:15.844Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-d5fdb9f7a199cdf3bbfc06b2d761364c7bcda5d0af25b97ff2276bb33e7b8ece-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689545167046
Jul 17 03:07:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:16.137Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"phy_rx_nobuf_total\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", dev=\"virtio_dign_s\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"mrzcpd-exporter\"}" timestamp=1689545166692
Jul 17 03:07:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:16.356Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_unicast_packets_transmitted_total\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_7\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545161230
Jul 17 03:07:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:16.359Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_gc_duration_seconds_count\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545166027
Jul 17 03:07:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:16.377Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens1f0np0\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545161230
Jul 17 03:07:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:16.419Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link_mode\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens7f1np1\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545161230
Jul 17 03:07:17 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:17.130Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dracut-initqueue.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689545167046
Jul 17 03:07:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:19.458Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-3a00f506971912e91a0f2421a4cd5f23450e9b733c343cbedf66427667057151.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"scope\"}" timestamp=1689545164356
Jul 17 03:07:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:19.905Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled_time\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-pushgateway-bd476bcf9-rlgwl\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"fe59f4a3-ab44-46a4-9b1d-4a02aedb6f14\"}" timestamp=1689545173949
Jul 17 03:07:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:19.922Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"51\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545177612
Jul 17 03:07:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:19.935Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"102\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545177612
Jul 17 03:07:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:19.982Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", fstype=\"ext4\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/var/lib/kubelet/pods/7fc8eb18-df9b-4e31-aa08-e338eee64b81/volume-subpaths/config-volume/telegraf/2\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545177612
Jul 17 03:07:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:20.000Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"phy_rx_nobuf_total\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", dev=\"ens7f1np1\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"mrzcpd-exporter\"}" timestamp=1689545177986
Jul 17 03:07:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:20.018Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"mdmonitor.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689545164356
Jul 17 03:07:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:20.293Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_inodes_free\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda3\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545166211
Jul 17 03:07:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:20.465Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db76\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545174840
Jul 17 03:07:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:20.512Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_observed_generation\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"local-path-provisioner\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545173949
Jul 17 03:07:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:20.528Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_spec_type\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"metrics-server\", type=\"ClusterIP\", uid=\"dec037a8-7e81-41d7-97ed-0c9c80cde541\"}" timestamp=1689545173949
Jul 17 03:07:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:20.664Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", container=\"coredns\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Read\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545168954
Jul 17 03:07:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:20.765Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-2199681b\\\\x2d3e95\\\\x2d4349\\\\x2d8d1f\\\\x2d0c2eb8c38c0f-volumes-kubernetes.io\\\\x7esecret-config.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689545164356
Jul 17 03:07:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:20.807Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-virtual-misc-rfkill.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689545164356
Jul 17 03:07:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:20.823Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-veth48720f3f.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689545164356
Jul 17 03:07:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:20.871Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"pcap-tsgx10\", asset_id=\"89\", condition=\"true\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"kube-multus-ds-sqrb7\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"ee8afe42-b7fb-4840-a9ee-3d0e7212c356\"}" timestamp=1689545173828
Jul 17 03:07:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:20.915Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", pod=\"tsg-traffic-engine-numa-1-firewall-5d7bfbbbfb-4wgxr\", project=\"TSG-OS\", project_id=\"7\", reason=\"NodeAffinity\", service=\"prometheus-kube-state-metrics\", uid=\"ab53e5a7-f22e-498d-b5af-535ae7359350\"}" timestamp=1689545173828
Jul 17 03:07:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:20.938Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"tmpfs\", name=\"data-var-lib-kubelet-pods-26d0147a\\\\x2dfc67\\\\x2d4990\\\\x2d96ac\\\\x2d7d6c8e980d26-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2dtzj5r.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689545170657
Jul 17 03:07:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:20.989Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"gssproxy.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"service\"}" timestamp=1689545170657
Jul 17 03:07:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:21.341Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"78\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545168263
Jul 17 03:07:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:21.355Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"9\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545168263
Jul 17 03:07:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:21.388Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cryptsetup-pre.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"target\"}" timestamp=1689545170657
Jul 17 03:07:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:21.437Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_writes_merged_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/run/user/1001\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545166211
Jul 17 03:07:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:21.559Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_tasks_max\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sshd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689545170657
Jul 17 03:07:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:21.662Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db199\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545174840
Jul 17 03:07:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:21.706Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"86\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545168263
Jul 17 03:07:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:21.826Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_multicast_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth3417e171\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545168263
Jul 17 03:07:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:21.839Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_timex_pps_jitter_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545168263
Jul 17 03:07:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:22.304Z caller=dedupe.go:112 component=remote level=error remote_name=52db6f url=http://10.10.10.159:10090/nz-agent/prometheus/proxy/api/v1/write msg="non-recoverable error" count=1000 exemplarCount=0 err="server returned HTTP status 400 : out of order sample"
Jul 17 03:07:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:22.566Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"druid_index_pending_task_num\", asset=\"msh-dellr740-olap004\", asset_id=\"17\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Spark-msh-dellr740-olap004\", endpoint_id=\"15\", module=\"NC-Spark\", module_id=\"47\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545182253
Jul 17 03:07:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:23.697Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_max_fds\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"getty@tty1.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689545178416
Jul 17 03:07:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:23.711Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"rtkit-daemon.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"dbus\", type=\"service\"}" timestamp=1689545178416
Jul 17 03:07:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:23.791Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_labels\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_endpoint=\"ipmi-exporter\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545180068
Jul 17 03:07:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:23.813Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_address_assign_type\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth0c6b6d72\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545174790
Jul 17 03:07:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:23.864Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda5\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", fstype=\"ext4\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/home\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545174790
Jul 17 03:07:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:23.876Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_files_free\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda5\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", fstype=\"ext4\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/home\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545174790
Jul 17 03:07:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:23.881Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Succeeded\", pod=\"helm-install-traefik-crd-tjh7p\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"2a496308-3fd5-494e-83f9-537bb34e5898\"}" timestamp=1689545180068
Jul 17 03:07:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:23.891Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_multicast_total\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"lo\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545174790
Jul 17 03:07:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:23.896Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx09\", asset_id=\"55\", cpu=\"94\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545174790
Jul 17 03:07:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:24.288Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"metrics-server\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Discard\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545178126
Jul 17 03:07:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:24.309Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_mapped_file\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"prometheus-server-configmap-reload\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545174414
Jul 17 03:07:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:24.457Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_protocol_type\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"vethcf071709\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545174790
Jul 17 03:07:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:24.473Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_errs_total\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"eno2\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545174790
Jul 17 03:07:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:24.697Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failcnt\", asset=\"msh-tsgx09\", asset_id=\"55\", container=\"prometheus-alertmanager\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545177258
Jul 17 03:07:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:24.721Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_info\", asset=\"msh-tsgx09\", asset_id=\"55\", created_by_kind=\"ReplicaSet\", created_by_name=\"ipmi-exporter-6f8464dd94\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", host_ip=\"10.10.20.169\", host_network=\"false\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", node=\"localhost\", nz_agent_id=\"3\", pod=\"ipmi-exporter-6f8464dd94-n7w69\", pod_ip=\"10.42.0.151\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"9e233e9f-de40-4ebd-abfb-e6360a2d9bb6\"}" timestamp=1689545180068
Jul 17 03:07:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:24.733Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dracut-pre-trigger.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689545178416
Jul 17 03:07:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:24.878Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_mapped_file\", asset=\"msh-tsgx09\", asset_id=\"55\", container=\"kube-multus\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545165466
Jul 17 03:07:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:24.888Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_limit_bytes\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"metrics-server\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545183423
Jul 17 03:07:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:25.002Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_writes_bytes_total\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545178209
Jul 17 03:07:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:25.010Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_info\", address=\"e4:3d:1a:dd:b5:03\", asset=\"pcap-tsgx12\", asset_id=\"117\", broadcast=\"ff:ff:ff:ff:ff:ff\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"eno36np3\", duplex=\"unknown\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operstate=\"down\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545180160
Jul 17 03:07:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:25.220Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"proc-sys-fs-binfmt_misc.automount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"automount\"}" timestamp=1689545171102
Jul 17 03:07:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:25.291Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_address_assign_type\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth7ba955f4\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545180160
Jul 17 03:07:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:25.311Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_id\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth7ba955f4\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545180160
Jul 17 03:07:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:25.394Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_drop_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"vethc0962f36\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545180160
Jul 17 03:07:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:25.439Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.grpc.v1.cri-sandboxes-9278e24d65ca2abe9945e75398d615ce969e9aab821b6c0ba4062251bd8e1a07-shm.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689545171102
Jul 17 03:07:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:25.865Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_mallocs_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545181795
Jul 17 03:07:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:25.925Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db63\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545181795
Jul 17 03:07:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:25.986Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-sda5.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689545171102
Jul 17 03:07:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:26.001Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_fifo_total\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth188af860\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545175810
Jul 17 03:07:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:26.158Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-coredump@0.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"service\"}" timestamp=1689545171102
Jul 17 03:07:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:26.568Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545175810
Jul 17 03:07:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:26.586Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"56\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545175810
Jul 17 03:07:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:26.626Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_spec_paused\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"prometheus-server\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545178744
Jul 17 03:07:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:26.629Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"overlay\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-3e809e3bc00c2d08bd95af19279421eb607cd2ee8b37525da9ec846407e68fb7-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689545173907
Jul 17 03:07:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:26.641Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"22\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545175810
Jul 17 03:07:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:26.776Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth765ebb62\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545175810
Jul 17 03:07:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:26.793Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"40\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545175810
Jul 17 03:07:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:27.025Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-infiniband-umad1.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689545173907
Jul 17 03:07:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:27.034Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"102\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545175810
Jul 17 03:07:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:27.058Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_hwmon_chip_names\", asset=\"twa-tsgx09\", asset_id=\"107\", chip=\"0000:60:03_1_0000:63:00_2\", chip_name=\"bnxt_en\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545175810
Jul 17 03:07:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:27.125Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"abrt-journal-core.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"service\"}" timestamp=1689545173907
Jul 17 03:07:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:27.244Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"86\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545175810
Jul 17 03:07:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:27.288Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"scrape_series_added\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_instance=\"10.42.0.82:9005\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"mrzcpd-exporter\"}" timestamp=1689545178295
Jul 17 03:07:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:27.321Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_speed_bytes\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"eno35np2\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545175810
Jul 17 03:07:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:27.377Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_file_descriptors\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"exporter\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545181099
Jul 17 03:07:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:27.687Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_created\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"systemd-exporter-7f549d9485\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545175682
Jul 17 03:07:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:27.728Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"17\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545175810
Jul 17 03:07:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:27.747Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_scrape_collector_success\", asset=\"msh-tsgx10\", asset_id=\"69\", collector=\"mdadm\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545182882
Jul 17 03:07:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:27.796Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"system-mlnx_interface_mgr.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"slice\"}" timestamp=1689545173907
Jul 17 03:07:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:28.086Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"2\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545182882
Jul 17 03:07:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:28.196Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db156\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545179446
Jul 17 03:07:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:28.419Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_info\", asset=\"msh-tsgx10\", asset_id=\"69\", cluster_ip=\"10.43.80.124\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"traefik\", uid=\"dbdae63f-5ac2-4c96-a503-e163087d3851\"}" timestamp=1689545175682
Jul 17 03:07:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:28.426Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_compressed_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens2f0np0\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545182882
Jul 17 03:07:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:28.462Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"107\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545182882
Jul 17 03:07:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:28.469Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_errs_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens7f0np0\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545182882
Jul 17 03:07:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:28.483Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"jaeger_tracer_sampler_queries_total\", agent_name=\"msh-kvm-nz001\", application=\"loki\", datacenter=\"MSH\", instance=\"127.0.0.1:13100\", result=\"err\"}" timestamp=1689545188327
Jul 17 03:07:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:28.583Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"84\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545182882
Jul 17 03:07:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:28.606Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"overlay\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-d15afe0c0439a396fd2f20cccb18316dde637440f279c6c608eb6894b0bddf9d-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689545174114
Jul 17 03:07:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:28.658Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"libstoragemgmt.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"simple\", type=\"service\"}" timestamp=1689545174114
Jul 17 03:07:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:28.838Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"32\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545182882
Jul 17 03:07:29 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:29.095Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-4a46604b58b75a9937b4aeb3b07fd5d20020186af7a669263bba5457079d9f06-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689545174114
Jul 17 03:07:29 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:29.122Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"2\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545182882
Jul 17 03:07:29 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:29.145Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_files_free\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"shm\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/1080914a00fa69374e5d41349cf40ef240e9b34f20fe33f9a143497c0f85cc19/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545182882
Jul 17 03:07:29 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:29.308Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"msh-tsgx10\", asset_id=\"69\", condition=\"Progressing\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"cm-policy-local-cache-redis-master\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"true\"}" timestamp=1689545175682
Jul 17 03:07:29 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:29.431Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"loki_kv_request_duration_seconds_bucket\", agent_name=\"msh-kvm-nz001\", application=\"loki\", datacenter=\"MSH\", instance=\"127.0.0.1:13100\", kv_name=\"scheduler\", le=\"0.025\", operation=\"GET\", role=\"primary\", status_code=\"200\", type=\"memberlist\"}" timestamp=1689545188327
Jul 17 03:07:30 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:30.926Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_user_seconds_total\", asset=\"msh-tsgx05\", asset_id=\"67\", container=\"kube-state-metrics\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545173011
Jul 17 03:07:30 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:30.963Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_daemonset_status_number_unavailable\", asset=\"msh-tsgx05\", asset_id=\"67\", daemonset=\"dign-client\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545184068
Jul 17 03:07:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:31.020Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx05\", asset_id=\"67\", container=\"coredns\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Async\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545171768
Jul 17 03:07:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:31.037Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"82\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545180939
Jul 17 03:07:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:31.052Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_dormant\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"vethef1c04ca\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545180939
Jul 17 03:07:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:31.479Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"twa-tsgx03\", asset_id=\"95\", container=\"prometheus-server-configmap-reload\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"running\"}" timestamp=1689545176649
Jul 17 03:07:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:31.592Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-ens4f2np2.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689545186907
Jul 17 03:07:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:31.654Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"30\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545179093
Jul 17 03:07:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:31.998Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db44\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545176711
Jul 17 03:07:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:32.006Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cryptsetup.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"target\"}" timestamp=1689545186907
Jul 17 03:07:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:32.030Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-1df6bd2b\\\\x2d0fea\\\\x2d4549\\\\x2d94c2\\\\x2dddadc3bbc3e4-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2d24sbj.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689545186907
Jul 17 03:07:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:32.034Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"32\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545180791
Jul 17 03:07:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:32.081Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_compressed_total\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens7f1np1\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545180791
Jul 17 03:07:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:32.101Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-update-done.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689545182644
Jul 17 03:07:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:32.102Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_frame_total\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth01f00b07\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545180791
Jul 17 03:07:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:32.150Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-461d1588\\\\x2d9ea2\\\\x2d41cf\\\\x2d8eca\\\\x2ddaa78ce6d555-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2d2gr89.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689545182644
Jul 17 03:07:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:32.192Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"coredns\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Write\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545166403
Jul 17 03:07:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:32.222Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"monit-stream\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Sync\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545175269
Jul 17 03:07:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:32.524Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545180605
Jul 17 03:07:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:32.589Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx05\", asset_id=\"99\", container=\"dign-client\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Sync\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545178404
Jul 17 03:07:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:32.610Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_labels\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"cm-policy-local-cache-redis-master-69967b5879-vlgl8\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"e8ef10b6-3745-4720-b3f4-f80080df3d45\"}" timestamp=1689545179894
Jul 17 03:07:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:32.645Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-59b8c2970a19b1f3bf5af892cc518f92b562af2a9375ecbf7f57fbc5d4ae0b58-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689545187691
Jul 17 03:07:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:32.665Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"swap.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"target\"}" timestamp=1689545187691
Jul 17 03:07:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:32.687Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"18\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545180939
Jul 17 03:07:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:32.764Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx05\", asset_id=\"99\", container=\"dign-client\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Read\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545178404
Jul 17 03:07:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:32.822Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_processes\", asset=\"twa-tsgx03\", asset_id=\"95\", container=\"configmap-reloader\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545176818
Jul 17 03:07:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:32.877Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"17\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545180605
Jul 17 03:07:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:32.900Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_virtual_memory_bytes\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dbus.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689545187691
Jul 17 03:07:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:32.906Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_last_seen\", asset=\"twa-tsgx05\", asset_id=\"99\", container=\"telegraf-security\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689545178566
Jul 17 03:07:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:32.917Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-burstable-pod8c5c1473_ca58_435f_a331_a241afdff2f9.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"slice\"}" timestamp=1689545187691
Jul 17 03:07:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:32.919Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_processes\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"telegraf-security\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689545170346
Jul 17 03:07:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:32.953Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"107\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545180791
Jul 17 03:07:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:33.047Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"dashboard-metrics-scraper\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Discard\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545181502
Jul 17 03:07:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:33.048Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_limit_bytes\", asset=\"twa-tsgx03\", asset_id=\"95\", container=\"dign-server-dns\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545183123
Jul 17 03:07:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:33.066Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Unknown\", pod=\"helm-install-traefik-crd-xvctc\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"037f9a49-f9c9-47d9-b6d3-eeb2a6dc570c\"}" timestamp=1689545190436
Jul 17 03:07:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:33.081Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_fifo_total\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens4f2np2\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545179093
Jul 17 03:07:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:33.130Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db62\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545176711
Jul 17 03:07:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:33.186Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_changes_total\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth39378f16\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545179093
Jul 17 03:07:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:33.214Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_net_dev_group\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth244c5093\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545179093
Jul 17 03:07:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:33.224Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_compressed_total\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"eno34np1\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545180605
Jul 17 03:07:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:33.269Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"56\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545180605
Jul 17 03:07:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:33.272Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_dormant\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth1ccea153\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545180791
Jul 17 03:07:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:33.288Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_multicast_packets_received_total\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_9\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545180791
Jul 17 03:07:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:33.310Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_protocol_type\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethc8144301\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545180605
Jul 17 03:07:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:33.371Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"emergency.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689545187691
Jul 17 03:07:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:33.373Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_labels\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"svclb-traefik-f8ab0b79-f6t4m\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"6c839a6b-757d-460b-acb3-20a2f3df359e\"}" timestamp=1689545179894
Jul 17 03:07:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:33.419Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"14\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545180605
Jul 17 03:07:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:33.438Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_type\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"chart-values-ipmi-exporter\", service=\"prometheus-kube-state-metrics\", type=\"Opaque\"}" timestamp=1689545179276
Jul 17 03:07:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:33.452Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_gc_duration_seconds\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", quantile=\"0\", service=\"ipmi-exporter\"}" timestamp=1689545183498
Jul 17 03:07:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:33.456Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_address_assign_type\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"eno33np0\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545180605
Jul 17 03:07:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:33.469Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"msh-tsgx11\", asset_id=\"111\", condition=\"unknown\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-delete-tsg-traffic-engine-numa-1-cxlvk\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"1d75a15d-f414-4a89-93b6-193080fb66a4\"}" timestamp=1689545179276
Jul 17 03:07:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:33.476Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"76\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545180791
Jul 17 03:07:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:33.489Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-18031228\\\\x2dabd9\\\\x2d478c\\\\x2dbf5c\\\\x2d0acdbff9f2e6-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2djvq9c.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689545186907
Jul 17 03:07:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:33.518Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"102\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545179093
Jul 17 03:07:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:33.524Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-bond0.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689545187691
Jul 17 03:07:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:33.539Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_created\", asset=\"msh-tsgx11\", asset_id=\"111\", configmap=\"chart-content-systemd-exporter\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545179276
Jul 17 03:07:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:33.557Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_spec_strategy_rollingupdate_max_surge\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"prometheus-alertmanager\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545179276
Jul 17 03:07:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:33.747Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"twa-tsgx03\", asset_id=\"95\", condition=\"true\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"local-path-provisioner-957fdf8bc-q8cxs\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"88109228-9032-4124-8904-a72a5d84bc7b\"}" timestamp=1689545190436
Jul 17 03:07:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:33.755Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_mtu_bytes\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth6bfabd57\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545180791
Jul 17 03:07:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:33.777Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx05\", asset_id=\"99\", container=\"kube-multus\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Read\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545175595
Jul 17 03:07:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:34.170Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas_available\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"metrics-server\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545179276
Jul 17 03:07:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:34.221Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_dormant\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens2f0np0\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545180605
Jul 17 03:07:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:34.243Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"69\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545180605
Jul 17 03:07:39 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:39.821Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_scaling_frequency_hertz\", asset=\"twa-dellr740-dt003\", asset_id=\"41\", cpu=\"30\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"OLAP-Node-Exporter-twa-dellr740-dt003\", endpoint_id=\"261\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"hbase_master,clickhouse_data,kafka,gohangout,zookeeper,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545198473
Jul 17 03:07:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:40.385Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_working_set_bytes\", asset=\"twa-tsgx04\", asset_id=\"97\", container=\"telegraf-security\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689545176381
Jul 17 03:07:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:40.873Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_fully_labeled_replicas\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-0-firewall-7cbb5bf99c\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545200034
Jul 17 03:07:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:41.001Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_cpu_period\", asset=\"twa-tsgx04\", asset_id=\"97\", container=\"telegraf\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689545188797
Jul 17 03:07:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:41.122Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_max_usage_bytes\", asset=\"twa-tsgx04\", asset_id=\"97\", container=\"firewall\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689545183009
Jul 17 03:07:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:41.176Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"27\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545191647
Jul 17 03:07:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:41.515Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_exceptions\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"IPC\"}" timestamp=1689545199758
Jul 17 03:07:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:41.524Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_numReferenceFiles\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"default\", project=\"OLAP\", project_id=\"3\", region=\"c584b1f1b8c9832ec73bbf2f56ff9a4d\", table=\"index_time_default\", type=\"index\"}" timestamp=1689545199758
Jul 17 03:07:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:41.543Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_avgStoreFileAge\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", namespace=\"SYSTEM\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"0e32bd2946de68f3772b0f5addf5bbbf\", table=\"LOG\", type=\"data\"}" timestamp=1689545199758
Jul 17 03:07:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:41.610Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_10_194_1_metric_incrementTime_max\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545199758
Jul 17 03:07:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:41.767Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"82\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545191647
Jul 17 03:07:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:41.811Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx04\", asset_id=\"97\", container=\"prometheus-server\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Total\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545178205
Jul 17 03:07:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:41.837Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-0db21c869d06a1d205562f64dfba081347e731cefec274c3f8bb1b08696eeacf.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"scope\"}" timestamp=1689545195764
Jul 17 03:07:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:41.843Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_state_id\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_8\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545191647
Jul 17 03:07:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:41.987Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_numReferenceFiles\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"assessment_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"77d5f82e5f12961775c07304d24c8707\", table=\"index_partfile_assessment_hos_bucket\", type=\"index\"}" timestamp=1689545199758
Jul 17 03:07:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:42.019Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_flushesQueuedCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"session_record_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"d27b71ef7b3d34babbe1b13c2cccd0bd\", table=\"index_filename_session_record_hos_bucket\", type=\"index\"}" timestamp=1689545199758
Jul 17 03:07:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:42.045Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_10_194_1_metric_incrementTime_num_ops\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545199758
Jul 17 03:07:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:42.229Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-pod8e8e0871_84d7_40cc_aa16_976ff516a351.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"slice\"}" timestamp=1689545195764
Jul 17 03:07:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:42.242Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-a5104a71fcffa4fdb02c774d48755d39d7fbc70c216860da9d967bb75977a92f-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689545195764
Jul 17 03:07:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:42.273Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_filteredReadRequestCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"rtp_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"c35ab1d140c31d8ea561a5a9503555b9\", table=\"index_time_rtp_hos_bucket\", type=\"index\"}" timestamp=1689545199758
Jul 17 03:07:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:42.400Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_CATALOG_metric_splitTime_99th_percentile\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545199758
Jul 17 03:07:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:42.445Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_up_changes_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"br_dign_s\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545196242
Jul 17 03:07:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:42.464Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_address_assign_type\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens4f1np1\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545191647
Jul 17 03:07:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:42.844Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", cpu=\"87\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545196242
Jul 17 03:07:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:42.862Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_fifo_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth9d1dd258\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545196242
Jul 17 03:07:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:43.174Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_sockets\", asset=\"pcap-tsgx01\", asset_id=\"71\", container=\"dign-server-dns\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545191359
Jul 17 03:07:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:43.190Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_metadata_resource_version\", asset=\"pcap-tsgx01\", asset_id=\"71\", configmap=\"sce-tsg-traffic-engine-numa-1\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545195154
Jul 17 03:07:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:43.205Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_owner\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", owner_is_controller=\"true\", owner_kind=\"ReplicaSet\", owner_name=\"tsg-traffic-engine-numa-1-firewall-69ff8c6fd4\", pod=\"tsg-traffic-engine-numa-1-firewall-69ff8c6fd4-fzrff\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"a38bc0a0-f54d-4548-b0f1-6472f0282ca8\"}" timestamp=1689545195154
Jul 17 03:07:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:43.306Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-sysusers.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689545200671
Jul 17 03:07:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:43.412Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"42\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545196242
Jul 17 03:07:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:43.435Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"49\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545196242
Jul 17 03:07:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:43.450Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", cpu=\"83\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545196242
Jul 17 03:07:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:43.467Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"19\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545196242
Jul 17 03:07:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:43.475Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"abrt-journal-core.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"service\"}" timestamp=1689545200671
Jul 17 03:07:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:43.486Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", cpu=\"46\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545196242
Jul 17 03:07:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:43.497Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_receive_switch_relay_errors_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"mlx5_2\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545196242
Jul 17 03:07:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:43.535Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"mrzcpd_hwdb_setup.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"oneshot\", type=\"service\"}" timestamp=1689545200671
Jul 17 03:07:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:43.609Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db77\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545194838
Jul 17 03:07:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:43.646Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-bond0.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689545200671
Jul 17 03:07:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:43.661Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_nf_conntrack_stat_found\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545196242
Jul 17 03:07:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:43.683Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", cpu=\"111\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545196242
Jul 17 03:07:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:43.696Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda5\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", fstype=\"ext4\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/data/var/lib/kubelet/pods/d20d4034-164e-4450-a851-056ee84ef97e/volume-subpaths/config-volume/firewall/1\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545196242
Jul 17 03:07:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:43.709Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"rpc-statd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689545200671
Jul 17 03:07:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:43.727Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_metadata_resource_version\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"kubernetes-dashboard-key-holder\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545195154
Jul 17 03:07:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:43.763Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db180\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545194838
Jul 17 03:07:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:43.764Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-netns-cni\\\\x2d5ac5231e\\\\x2d7fd2\\\\x2d8fb6\\\\x2df179\\\\x2d25bd5e32babe.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689545200671
Jul 17 03:07:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:43.822Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_max_fds\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"hasp_monitor.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689545200671
Jul 17 03:07:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:43.839Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_virtual_memory_bytes\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"getty@tty1.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689545200671
Jul 17 03:07:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:43.860Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_address_assign_type\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"vethdd660f18\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545196242
Jul 17 03:07:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:43.874Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", cpu=\"2\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545196242
Jul 17 03:07:44 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:44.099Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_complete\", asset=\"pcap-tsgx01\", asset_id=\"71\", condition=\"false\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", job_name=\"helm-install-traefik\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545195154
Jul 17 03:07:44 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:44.273Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_nf_conntrack_stat_early_drop\", asset=\"msh-kvm-cm001\", asset_id=\"3\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"CM-Node-Exporter-msh-kvm-cm001\", endpoint_id=\"391\", module=\"CM-Node-Exporter\", module_id=\"75\", nz_agent_id=\"3\", parent_asset=\"msh-dellr740-pve001\", parent_asset_id=\"1\", project=\"Central-Management\", project_id=\"5\"}" timestamp=1689545204235
Jul 17 03:07:48 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:48.922Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_storeCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"assessment_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"a7a5c13a400401472e2d36778e8ca3a1\", table=\"index_time_assessment_hos_bucket\", type=\"index\"}" timestamp=1689545207439
Jul 17 03:07:48 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:48.943Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_lastMajorCompactionAge\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"troubleshooting_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"5e50514d4b253a35692439dd5a171c2f\", table=\"index_time_troubleshooting_hos_bucket\", type=\"index\"}" timestamp=1689545207439
Jul 17 03:07:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:48.998Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_3_metric_splitTime_99th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545207439
Jul 17 03:07:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:49.251Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_3_metric_compactionOutputSize_98th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545207439
Jul 17 03:07:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:49.264Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_hbase_table_namespace_metric_splitTime_90th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545207439
Jul 17 03:07:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:49.364Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_4_metric_majorCompactionTime_num_ops\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545207439
Jul 17 03:07:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:49.384Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_4_metric_scanSize_99_9th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545207439
Jul 17 03:07:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:49.485Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"twa-tsgx10\", asset_id=\"109\", cpu=\"89\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545201797
Jul 17 03:07:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:49.499Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"dashboard-metrics-scraper\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Write\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545196623
Jul 17 03:07:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:49.518Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_compressed_total\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens3f0np0\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545201797
Jul 17 03:07:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:49.620Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db11\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545206625
Jul 17 03:07:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:49.660Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_bytes_total\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", interface=\"ens7f1np1\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545196356
Jul 17 03:07:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:49.697Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"twa-tsgx10\", asset_id=\"109\", condition=\"true\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"traefik-64f55bb67d-lf2mv\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"5af7a30e-36d6-4201-b9cc-f18bde92c892\"}" timestamp=1689545199668
Jul 17 03:07:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:49.740Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_scrape_collector_duration_seconds\", asset=\"twa-tsgx10\", asset_id=\"109\", collector=\"powersupplyclass\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545201797
Jul 17 03:07:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:49.747Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_1_metric_scanTime_25th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545207439
Jul 17 03:07:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:49.814Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_metadata_resource_version\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"kubernetes-dashboard-certs\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545201568
Jul 17 03:07:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:49.856Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"scrape_samples_scraped\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_instance=\"10.42.0.80:9121\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545207556
Jul 17 03:07:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:49.890Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx10\", asset_id=\"109\", cpu=\"77\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545201797
Jul 17 03:07:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:49.908Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_hbase_table_namespace_metric_putBatchTime_99_9th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545207439
Jul 17 03:07:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:49.914Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"94\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545201797
Jul 17 03:07:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:49.931Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx10\", asset_id=\"109\", cpu=\"35\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545201797
Jul 17 03:07:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:49.947Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"rsyslog.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"service\"}" timestamp=1689545201057
Jul 17 03:07:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:49.959Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_2_metric_majorCompactionInputSize_95th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545207439
Jul 17 03:07:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:49.976Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_address_not_ready\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_endpoint=\"prometheus-node-exporter\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545199668
Jul 17 03:07:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:50.021Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"39\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545201797
Jul 17 03:07:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:50.024Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_read_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/data/var/lib/kubelet/pods/d4457315-9a37-45e8-b7b6-d04b15cea130/volumes/kubernetes.io~projected/kube-api-access-dg6lc\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545204523
Jul 17 03:07:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:50.041Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_sector_writes_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"overlay_0-141\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545204523
Jul 17 03:07:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:50.059Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_daemonset_metadata_generation\", asset=\"twa-tsgx08\", asset_id=\"105\", daemonset=\"kube-multus-ds\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545201568
Jul 17 03:07:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:50.063Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_swap_limit_bytes\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"prometheus-pushgateway\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545199050
Jul 17 03:07:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:50.069Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_hwmon_chip_names\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", chip=\"platform_coretemp_0\", chip_name=\"coretemp\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-olap003\", endpoint_id=\"235\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"yarn_resourcemanager,zookeeper,nacos,kafka,spark,druid_data,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545209715
Jul 17 03:07:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:50.104Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_compressed_total\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens8f1np1\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545201797
Jul 17 03:07:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:50.111Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_hwmon_temp_celsius\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", chip=\"platform_coretemp_1\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-olap003\", endpoint_id=\"235\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"yarn_resourcemanager,zookeeper,nacos,kafka,spark,druid_data,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\", sensor=\"temp5\"}" timestamp=1689545209715
Jul 17 03:07:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:50.153Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_session_record_hos_bucket_metric_majorCompactionInputFileCount_99_9th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545207439
Jul 17 03:07:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:50.184Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"88\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545201797
Jul 17 03:07:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:50.211Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2dpath-pci\\\\x2d0000:81:00.0\\\\x2dscsi\\\\x2d0:2:0:0\\\\x2dpart4.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689545201057
Jul 17 03:07:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:50.253Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"26\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545206709
Jul 17 03:07:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:50.310Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_hwmon_sensor_label\", asset=\"twa-tsgx10\", asset_id=\"109\", chip=\"pci0000:00_0000:00:18_3\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", label=\"tccd8\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", sensor=\"temp10\", service=\"prometheus-node-exporter\"}" timestamp=1689545201797
Jul 17 03:07:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:50.320Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_gc_duration_seconds\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-olap003\", endpoint_id=\"235\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"yarn_resourcemanager,zookeeper,nacos,kafka,spark,druid_data,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\", quantile=\"0.25\"}" timestamp=1689545209715
Jul 17 03:07:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:50.328Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_unicast_packets_transmitted_total\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545201797
Jul 17 03:07:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:50.339Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_xfs_directory_operation_remove_total\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"sdb1\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-olap003\", endpoint_id=\"235\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"yarn_resourcemanager,zookeeper,nacos,kafka,spark,druid_data,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545209715
Jul 17 03:07:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:50.346Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"virtio_dign_c\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545206709
Jul 17 03:07:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:50.358Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-olap003\", endpoint_id=\"235\", module=\"OLAP-Node-Exporter\", module_id=\"51\", name=\"11\", nz_agent_id=\"3\", olap_node_exporter=\"yarn_resourcemanager,zookeeper,nacos,kafka,spark,druid_data,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\", type=\"Processor\"}" timestamp=1689545209715
Jul 17 03:07:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:50.380Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_errors_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", interface=\"ens1f1np1\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545204523
Jul 17 03:07:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:50.411Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"120\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545206709
Jul 17 03:07:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:50.420Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", cpu=\"27\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-olap003\", endpoint_id=\"235\", mode=\"system\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"yarn_resourcemanager,zookeeper,nacos,kafka,spark,druid_data,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545209715
Jul 17 03:07:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:50.431Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dracut-shutdown.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"oneshot\", type=\"service\"}" timestamp=1689545206554
Jul 17 03:07:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:50.446Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_status_succeeded\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", job_name=\"helm-install-systemd-exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545201568
Jul 17 03:07:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:50.460Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx10\", asset_id=\"109\", cpu=\"83\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545201797
Jul 17 03:07:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:50.471Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_init_container_status_waiting\", asset=\"twa-tsgx08\", asset_id=\"105\", container=\"firewall-init\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", pod=\"tsg-traffic-engine-numa-0-firewall-8c9c7fc95-6rm9z\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"207ceb95-e1ef-4b7e-a587-0a6495b0f134\"}" timestamp=1689545201568
Jul 17 03:07:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:50.498Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_sector_writes_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"overlay_0-195\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545204523
Jul 17 03:07:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:50.517Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_194_4_metric_flushMemstoreSize_75th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545207439
Jul 17 03:07:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:50.582Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"61\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545206709
Jul 17 03:07:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:50.583Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_info\", asset=\"twa-tsgx08\", asset_id=\"105\", configmap=\"certstore-tsg-traffic-engine-numa-1\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545201568
Jul 17 03:07:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:50.596Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"prometheus-server\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Read\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545197533
Jul 17 03:07:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:50.604Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_complete\", asset=\"twa-tsgx08\", asset_id=\"105\", condition=\"false\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", job_name=\"helm-install-traefik\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545201568
Jul 17 03:07:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:50.604Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Failed\", pod=\"coredns-766944d45b-dstvr\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"b52de515-fecd-439f-b56f-e8953f782519\"}" timestamp=1689545199668
Jul 17 03:07:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:50.616Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_status_active\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", job_name=\"helm-install-tsg-traffic-engine-numa-0\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545201568
Jul 17 03:07:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:50.618Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_complete\", asset=\"twa-tsgx08\", asset_id=\"105\", condition=\"unknown\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", job_name=\"helm-install-traefik\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545201568
Jul 17 03:07:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:50.638Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_down_changes_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"lo\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545206709
Jul 17 03:07:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:50.642Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"108\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545201797
Jul 17 03:07:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:50.655Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_frees_total\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-olap003\", endpoint_id=\"235\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"yarn_resourcemanager,zookeeper,nacos,kafka,spark,druid_data,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545209715
Jul 17 03:07:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:50.658Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_scaling_frequency_hertz\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"70\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545206709
Jul 17 03:07:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:50.693Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_up\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"eno2\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545206709
Jul 17 03:07:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:50.695Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:00-0000:00:01.1-0000:01:00.0-net-ens1f0np0.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689545206554
Jul 17 03:07:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:50.718Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_fifo_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethbec6d7d4\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545206709
Jul 17 03:07:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:50.728Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_owner\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", owner_is_controller=\"true\", owner_kind=\"Deployment\", owner_name=\"tsg-traffic-engine-numa-1-firewall\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-1-firewall-6b6bf4679c\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545201568
Jul 17 03:07:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:50.731Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_node_status_capacity\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", node=\"localhost\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", resource=\"hugepages_1Gi\", service=\"prometheus-kube-state-metrics\", unit=\"byte\"}" timestamp=1689545201568
Jul 17 03:07:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:50.746Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_frequency_max_hertz\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"42\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545206709
Jul 17 03:07:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:50.747Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_frequency_min_hertz\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"2\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545206709
Jul 17 03:07:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:50.758Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"19\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545206709
Jul 17 03:07:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:50.786Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-virtio_dign_c.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689545201057
Jul 17 03:07:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:50.825Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"110\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545206709
Jul 17 03:07:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:50.828Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_time_clocksource_available_info\", asset=\"twa-tsgx08\", asset_id=\"105\", clocksource=\"tsc\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"0\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545206709
Jul 17 03:07:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:50.828Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_state_id\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_11\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545206709
Jul 17 03:07:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:50.843Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_queue_length\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth019f8c97\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545206709
Jul 17 03:07:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:50.845Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_packets_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth2ffdece6\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545206709
Jul 17 03:07:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:50.850Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_oom_events_total\", asset=\"twa-tsgx08\", asset_id=\"105\", container=\"local-path-provisioner\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545196391
Jul 17 03:07:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:50.870Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", state=\"iowaiting\"}" timestamp=1689545196730
Jul 17 03:07:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:50.909Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"85\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545206709
Jul 17 03:07:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:50.921Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_gc_duration_seconds_count\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-olap003\", endpoint_id=\"235\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"yarn_resourcemanager,zookeeper,nacos,kafka,spark,druid_data,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545209715
Jul 17 03:07:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:50.936Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.grpc.v1.cri-sandboxes-a379a67869d57aedcb0526bd2cb5e1a6fe7faed6994dcda2a2fc2426d6838604-shm.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689545206554
Jul 17 03:07:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:50.985Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_netstat_Tcp_InErrs\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545206709
Jul 17 03:07:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:50.999Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_frame_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"cnibr0\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545206709
Jul 17 03:07:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:51.015Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"88\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545206709
Jul 17 03:07:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:51.914Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"116\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545206709
Jul 17 03:07:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:51.916Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-263893bbd46a78b70b8e7c27d1451b8c3caed887f9ae82ecc6ad76e471a2aa0c-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689545206554
Jul 17 03:07:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:51.937Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_readonly\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda4\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", fstype=\"ext4\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545206709
Jul 17 03:07:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:52.490Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_last_terminated_reason\", asset=\"pcap-tsgx11\", asset_id=\"115\", container=\"mrapm-device\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"mrzcpd-exporter-5cf7f967cc-8bhqh\", project=\"TSG-OS\", project_id=\"7\", reason=\"Unknown\", service=\"prometheus-kube-state-metrics\", uid=\"1553814d-54e0-4f9e-b251-bfcf1ade925f\"}" timestamp=1689545204478
Jul 17 03:07:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:52.513Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", phase=\"Running\", pod=\"dign-client-gmxdm\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"4aff81f6-8efe-43da-b4c0-830d2bd4ff96\"}" timestamp=1689545204478
Jul 17 03:07:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:52.588Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_reservation_limit_bytes\", asset=\"pcap-tsgx11\", asset_id=\"115\", container=\"traefik\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545210039
Jul 17 03:07:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:52.620Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-podf2baf82c_b91b_46ec_8a59_8f55f1b86285.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"slice\"}" timestamp=1689545205496
Jul 17 03:07:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:52.640Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-e8f2ba66797d0142a62fdca9f99c1fbd1abbb542730104fba133927086d701b9-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689545205496
Jul 17 03:07:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:52.809Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", cpu=\"50\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545209337
Jul 17 03:07:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:52.903Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_cpu_seconds_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-burstable-podac103daf_2fd5_4026_ae82_9b9eca69f2cd.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"slice\"}" timestamp=1689545205496
Jul 17 03:07:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:52.984Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"atd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"service\"}" timestamp=1689545205496
Jul 17 03:07:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:53.026Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx08\", asset_id=\"65\", container=\"coredns\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Async\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545195617
Jul 17 03:07:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:53.057Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"msh-tsgx08\", asset_id=\"65\", condition=\"unknown\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-systemd-exporter-k9pkr\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"8db4e66c-b88f-4e1e-a738-4f13414c9d3b\"}" timestamp=1689545208950
Jul 17 03:07:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:53.064Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx08\", asset_id=\"65\", cpu=\"53\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545208328
Jul 17 03:07:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:53.101Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sshd-keygen@rsa.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689545212135
Jul 17 03:07:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:53.105Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"overlay\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-a3794eed0109c097b738cddacfd8dbf0876e211c49268030380158dee8d6da8e-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689545205496
Jul 17 03:07:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:53.121Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-update-done.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689545205496
Jul 17 03:07:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:53.168Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"polkit.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"service\"}" timestamp=1689545205496
Jul 17 03:07:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:53.201Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_persistentvolumeclaim_created\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", persistentvolumeclaim=\"prometheus-alertmanager\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545208950
Jul 17 03:07:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:53.226Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_packets_received_total\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_9\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545208328
Jul 17 03:07:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:53.247Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_transmit_wait_total\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_8\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545203391
Jul 17 03:07:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:53.262Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"rpc_pipefs.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"target\"}" timestamp=1689545212135
Jul 17 03:07:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:53.267Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_speed_bytes\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens8f0np0\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545203391
Jul 17 03:07:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:53.303Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx08\", asset_id=\"65\", container=\"dashboard-metrics-scraper\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Discard\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545195199
Jul 17 03:07:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:53.340Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-initctl.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"service\"}" timestamp=1689545212135
Jul 17 03:07:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:53.352Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db166\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545210108
Jul 17 03:07:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:53.361Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"vdo.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"service\"}" timestamp=1689545212135
Jul 17 03:07:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:53.431Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_cpu_seconds_total\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-udevd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689545212135
Jul 17 03:07:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:53.439Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_errs_total\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth5a5e6ccb\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545203391
Jul 17 03:07:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:53.448Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"ext4\", name=\"var-lib-kubelet-pods-a7dd7c83\\\\x2d1f08\\\\x2d4d39\\\\x2da9a0\\\\x2deff0bdbd01be-volume\\\\x2dsubpaths-config\\\\x2dvolume-telegraf-2.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689545212135
Jul 17 03:07:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:53.462Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"tmpfs\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/data/var/lib/kubelet/pods/e3fb9886-6958-4504-8536-21d26d231bc8/volumes/kubernetes.io~projected/kube-api-access-8zdwv\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545203391
Jul 17 03:07:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:53.497Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link_mode\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"vethd3b50248\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545203391
Jul 17 03:07:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:53.516Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-ac7de96c1bdae86889e3ec214aaa0ccbf3c6c5d57fd25bea7f501cb63a497681.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"scope\"}" timestamp=1689545211179
Jul 17 03:07:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:53.522Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-b9a87a56\\\\x2d93cb\\\\x2d41e5\\\\x2d883f\\\\x2dd940809bc904-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2d8mvvx.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689545212135
Jul 17 03:07:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:53.594Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_device_id\", asset=\"pcap-dellr740-dt002\", asset_id=\"33\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth6201062\", endpoint=\"NC-Gohangout-pcap-dellr740-dt002\", endpoint_id=\"173\", module=\"DC-Gohangout\", module_id=\"35\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545212666
Jul 17 03:07:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:53.603Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_metadata_generation\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"dashboard-metrics-scraper-5b94897dc9\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545208663
Jul 17 03:07:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:53.656Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_tasks_max\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-rfkill.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689545212135
Jul 17 03:07:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:53.669Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_sockets\", asset=\"msh-tsgx12\", asset_id=\"113\", container=\"kube-state-metrics\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545192772
Jul 17 03:07:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:53.671Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_drop_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens7f0np0\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545199409
Jul 17 03:07:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:53.691Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failcnt\", asset=\"msh-tsgx08\", asset_id=\"65\", container=\"telegraf-security\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689545192581
Jul 17 03:07:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:53.711Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_readonly\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"shm\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/6a42a80abfb46506b7f8eeb19e19bd7c41d29e6f92ec4fd41a22714d64242498/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545199409
Jul 17 03:07:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:53.715Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"112\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545203391
Jul 17 03:07:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:53.730Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_link_error_recovery_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"mlx5_2\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545199409
Jul 17 03:07:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:53.731Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_bytes_total\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth014c7ff4\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545208328
Jul 17 03:07:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:53.761Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"msh-tsgx08\", asset_id=\"65\", container=\"ipmi-exporter\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"running\"}" timestamp=1689545185888
Jul 17 03:07:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:53.774Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx08\", asset_id=\"65\", cpu=\"9\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545208328
Jul 17 03:07:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:53.776Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx12\", asset_id=\"113\", cpu=\"56\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545203391
Jul 17 03:07:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:53.832Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_state_started\", asset=\"pcap-tsgx07\", asset_id=\"85\", container=\"dign-server-dns\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"dign-server-4j9wh\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"70d61467-349b-47d9-b813-5d6a3af28cef\"}" timestamp=1689545202058
Jul 17 03:07:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:53.874Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:80-0000:80:01.1-0000:81:00.0-host0-target0:2:0-0:2:0:0-block-sda-sda2.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689545212135
Jul 17 03:07:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:53.887Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_link_downed_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"mlx5_1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545199409
Jul 17 03:07:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:53.913Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db198\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545210108
Jul 17 03:07:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:53.953Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_info\", asset=\"msh-tsgx12\", asset_id=\"113\", configmap=\"kube-apiserver-legacy-service-account-token-tracking\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545208663
Jul 17 03:07:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:53.959Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", phase=\"Running\", pod=\"dashboard-metrics-scraper-5b94897dc9-vm626\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"db0f9f5a-e16e-4bbf-ba42-94fb5a5a260d\"}" timestamp=1689545208663
Jul 17 03:07:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:53.972Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", phase=\"Running\", pod=\"tsg-traffic-engine-numa-1-firewall-89d4cc897-2jc89\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"8ba148c1-f25f-4fa4-82b3-03b9d5a8ace6\"}" timestamp=1689545208663
Jul 17 03:07:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:53.976Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx12\", asset_id=\"113\", cpu=\"66\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545203391
Jul 17 03:07:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:53.993Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_vl15_dropped_total\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_0\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545203391
Jul 17 03:07:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:54.010Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", state=\"stopped\"}" timestamp=1689545205786
Jul 17 03:07:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:54.026Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_virtual_memory_bytes\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"gssproxy.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689545211179
Jul 17 03:07:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:54.027Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_address_available\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_endpoint=\"systemd-exporter\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545208950
Jul 17 03:07:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:54.036Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-ask-password-plymouth.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689545211179
Jul 17 03:07:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:54.039Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_labels\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"traefik-64f55bb67d\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545208950
Jul 17 03:07:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:54.044Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_spec_paused\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"mrzcpd-exporter\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545208663
Jul 17 03:07:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:54.054Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_type\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"chart-values-systemd-exporter\", service=\"prometheus-kube-state-metrics\", type=\"Opaque\"}" timestamp=1689545208950
Jul 17 03:07:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:54.087Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"scrape_duration_seconds\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_instance=\"10.10.10.167:9008\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689545200884
Jul 17 03:07:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:54.090Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"system-mlnx_interface_mgr.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"slice\"}" timestamp=1689545200884
Jul 17 03:07:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:54.100Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:e0-0000:e0:03.1-0000:e2:00.1-infiniband-mlx5_11.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689545211179
Jul 17 03:07:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:54.133Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_limit_bytes\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545190574
Jul 17 03:07:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:54.141Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"plymouth-start.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"service\"}" timestamp=1689545212135
Jul 17 03:07:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:54.144Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"java_lang_MemoryPool_PeakUsage_committed\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-Yarn-msh-dellr740-dt001-NodeManager\", endpoint_id=\"351\", module=\"DC-Yarn\", module_id=\"1\", name=\"Metaspace\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545213514
Jul 17 03:07:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:54.146Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_annotations\", asset=\"msh-tsgx12\", asset_id=\"113\", configmap=\"sapp-tsg-traffic-engine-numa-0\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545208663
Jul 17 03:07:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:54.162Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_transmit_wait_total\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545203391
Jul 17 03:07:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:54.183Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cpu=\"15\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545199409
Jul 17 03:07:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:54.201Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx12\", asset_id=\"113\", cpu=\"56\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545203391
Jul 17 03:07:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:54.215Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_files\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"shm\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/427f69c810b459e27c5365027a549d066fdee35800ea00efe8cec92e50894533/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545203391
Jul 17 03:07:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:54.231Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_drop_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"eno34np1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545199409
Jul 17 03:07:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:54.252Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-79b32c04c82b6fe45e808fddfb58de8889e04656807045b8cb932e791bc81c61.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"scope\"}" timestamp=1689545200884
Jul 17 03:07:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:54.271Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_system_seconds_total\", asset=\"msh-tsgx08\", asset_id=\"65\", container=\"telegraf-security\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689545195455
Jul 17 03:07:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:54.314Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_labels\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-server\", uid=\"adbf05b1-b54e-4734-868a-0e1bc8f1f924\"}" timestamp=1689545208950
Jul 17 03:07:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:54.347Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-a1975ded\\\\x2d75ad\\\\x2d4725\\\\x2d8c15\\\\x2d1d1e0cdcaa0a-volumes-kubernetes.io\\\\x7esecret-cm\\\\x2dcache\\\\x2dextra\\\\x2dsecrets.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689545212135
Jul 17 03:07:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:54.412Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-fsck-root.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"oneshot\", type=\"service\"}" timestamp=1689545200884
Jul 17 03:07:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:54.419Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx08\", asset_id=\"65\", container=\"coredns\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Read\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545195617
Jul 17 03:07:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:54.601Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"init.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"scope\"}" timestamp=1689545200884
Jul 17 03:07:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:54.666Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"tsg_master_sce\", app_name=\"tsg_master\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"traffic-engine\", line_name=\"activing_success\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", table_name=\"ctrl_service\", vsysId=\"1\"}" timestamp=1689545200470
Jul 17 03:07:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:54.714Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_multicast_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens8f0np0\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545199409
Jul 17 03:07:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:54.749Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-a7cfa15dbba05667dbee87d84df2a784e59d9c20bb68873b5911c09871e25d79.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"scope\"}" timestamp=1689545200884
Jul 17 03:07:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:54.792Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_node_status_condition\", asset=\"pcap-tsgx07\", asset_id=\"85\", condition=\"DiskPressure\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", node=\"localhost\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"unknown\"}" timestamp=1689545202058
Jul 17 03:07:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:55.471Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cpu=\"12\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545199409
Jul 17 03:07:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:55.510Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_drop_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"vethf1e4e7f4\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545199409
Jul 17 03:07:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:55.532Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cpu=\"70\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545199409
Jul 17 03:07:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:55.586Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"cadvisor\", failure_type=\"pgfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"hierarchy\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689545201688
Jul 17 03:07:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:56.411Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_system_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", container=\"local-path-provisioner\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545202931
Jul 17 03:07:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:56.447Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db135\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545206475
Jul 17 03:07:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:56.636Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"3\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545205883
Jul 17 03:07:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:56.771Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"54\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545205883
Jul 17 03:07:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:56.817Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sshd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"notify\", type=\"service\"}" timestamp=1689545203372
Jul 17 03:07:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:56.848Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"network-online.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"target\"}" timestamp=1689545203372
Jul 17 03:07:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:57.197Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_netstat_Tcp_RetransSegs\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545205883
Jul 17 03:07:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:57.252Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_file_descriptors\", asset=\"pcap-tsgx03\", asset_id=\"73\", container=\"dign-server-dns\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545208188
Jul 17 03:07:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:57.293Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_annotations\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_endpoint=\"traefik\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545213389
Jul 17 03:07:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:57.346Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"phy_tx_packets_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", dev=\"ens8f1np1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"mrzcpd-exporter\"}" timestamp=1689545209991
Jul 17 03:07:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:57.509Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-dellr740-dt003\", asset_id=\"9\", cpu=\"44\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-dt003\", endpoint_id=\"229\", mode=\"user\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"hbase_master,clickhouse_data,kafka,gohangout,zookeeper,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545217407
Jul 17 03:07:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:57.607Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"cadvisor\", failure_type=\"pgfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"container\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689545201023
Jul 17 03:07:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:57.613Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-900223f6\\\\x2d7495\\\\x2d4cd9\\\\x2db023\\\\x2d184522fd9a95-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2dwmk4d.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689545203372
Jul 17 03:07:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:57.626Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-podd1430253_8c7e_449d_9f2a_dd552c402bae.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"slice\"}" timestamp=1689545203372
Jul 17 03:07:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:59.654Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545209083
Jul 17 03:07:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:59.656Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_udp_queues\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", ip=\"v4\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", queue=\"tx\", service=\"prometheus-node-exporter\"}" timestamp=1689545209083
Jul 17 03:07:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:59.678Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-ba9b06b98dd505edff5e19ed23b57302f9c9fd6bc494c517f37620714ac8b0b8-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689545204879
Jul 17 03:07:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:59.698Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"87\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545209083
Jul 17 03:07:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:59.741Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-ens2f1np1.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689545204879
Jul 17 03:07:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:07:59.788Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"24\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545209083
Jul 17 03:08:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:00.031Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2did-scsi\\\\x2d362cea7f085d4ec002c2b67bee386ff9d\\\\x2dpart4.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689545204879
Jul 17 03:08:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:00.118Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_info\", asset=\"msh-tsgx01\", asset_id=\"51\", cluster_ip=\"10.43.125.173\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"dashboard-metrics-scraper\", uid=\"3c14f3c0-7875-4635-8ba1-9ea1bec99363\"}" timestamp=1689545207425
Jul 17 03:08:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:00.237Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"2\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545209083
Jul 17 03:08:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:00.253Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_device_id\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens2f1np1\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545209083
Jul 17 03:08:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:00.331Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_errs_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth3c9a820a\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545209083
Jul 17 03:08:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:00.454Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens4f0np0\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545209083
Jul 17 03:08:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:00.540Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_metadata_resource_version\", asset=\"msh-tsgx01\", asset_id=\"51\", configmap=\"chart-content-ipmi-exporter\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545207425
Jul 17 03:08:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:00.697Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_memory_used_bytes\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545205071
Jul 17 03:08:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:00.708Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"36\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545209083
Jul 17 03:08:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:00.873Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"msh-tsgx01\", asset_id=\"51\", condition=\"Progressing\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"dashboard-metrics-scraper\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"false\"}" timestamp=1689545207425
Jul 17 03:08:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:00.887Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_created\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"prometheus-pushgateway-bd476bcf9\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545207425
Jul 17 03:08:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:00.887Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"msh-tsgx01\", asset_id=\"51\", container=\"kube-multus\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"running\"}" timestamp=1689545212946
Jul 17 03:08:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:00.914Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Failed\", pod=\"helm-install-tsg-traffic-engine-numa-0-j6fjb\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"0744435a-4d10-4030-a4e0-c03d8d48b916\"}" timestamp=1689545207425
Jul 17 03:08:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:00.969Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"19\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545209083
Jul 17 03:08:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:00.982Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_multicast_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth8bc626f9\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545209083
Jul 17 03:08:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:01.008Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_vl15_dropped_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_1\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545209083
Jul 17 03:08:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:02.541Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_mapped_file\", asset=\"msh-tsgx02\", asset_id=\"63\", container=\"dign-server-web\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545208938
Jul 17 03:08:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:02.800Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_transmit_wait_total\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_4\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545218318
Jul 17 03:08:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:03.551Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"prometheus_target_sync_length_seconds_count\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", instance=\"10.10.20.159:10090\", scrape_job=\"277\", type=\"global\"}" timestamp=1689545222990
Jul 17 03:08:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:03.689Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"msh-tsgx02\", asset_id=\"63\", cpu=\"78\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545218318
Jul 17 03:08:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:03.886Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_errors_total\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", interface=\"ens7f0np0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545211316
Jul 17 03:08:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:03.908Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", state=\"sleeping\"}" timestamp=1689545205865
Jul 17 03:08:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:04.202Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"msh-tsgx02\", asset_id=\"63\", container=\"telegraf\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", state=\"iowaiting\"}" timestamp=1689545203920
Jul 17 03:08:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:04.262Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_carrier_total\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"bond0\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545218318
Jul 17 03:08:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:04.276Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"initrd-root-fs.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"target\"}" timestamp=1689545208447
Jul 17 03:08:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:04.371Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_readonly\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"shm\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/3dc9560d34854d0ab10b492557f9080698dc968713dde4efdee2f2f982e585a6/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545218318
Jul 17 03:08:05 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:05.125Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"prometheus_target_scrape_pool_target_limit\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", instance=\"10.10.20.159:10090\", scrape_job=\"439\", type=\"global\"}" timestamp=1689545222990
Jul 17 03:08:05 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:05.658Z caller=dedupe.go:112 component=remote level=info remote_name=52db6f url=http://10.10.10.159:10090/nz-agent/prometheus/proxy/api/v1/write msg="Dropped sample for series that was not explicitly dropped via relabelling" ref=4436650
Jul 17 03:08:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:06.689Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_labels\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-1-firewall-5cb6f857dc\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545221227
Jul 17 03:08:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:06.779Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_errs_total\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"br_dign_s\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545223910
Jul 17 03:08:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:06.896Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_metadata_resource_version\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.tsg-traffic-engine-numa-1.v3\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545221227
Jul 17 03:08:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:07.020Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_cpu_seconds_total\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-pod377e59e7_1c6e_433a_a40c_92bbd7c96064.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"slice\"}" timestamp=1689545213582
Jul 17 03:08:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:07.034Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_speed_bytes\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth07e82765\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545223910
Jul 17 03:08:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:07.043Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dracut-shutdown.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"service\"}" timestamp=1689545213582
Jul 17 03:08:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:07.070Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db36\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545218290
Jul 17 03:08:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:07.089Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filefd_maximum\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545223910
Jul 17 03:08:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:07.501Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_up\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethb0e8bdf7\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545223910
Jul 17 03:08:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:07.517Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"gpm.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689545213582
Jul 17 03:08:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:07.530Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-pod377e59e7_1c6e_433a_a40c_92bbd7c96064.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"slice\"}" timestamp=1689545213582
Jul 17 03:08:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:07.783Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_dormant\", asset=\"pcap-tsgx06\", asset_id=\"81\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"lo\", endpoint=\"TSG-OS-pcap-tsgx06\", endpoint_id=\"475\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545216740
Jul 17 03:08:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:08.006Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_receive_remote_physical_errors_total\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_7\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545223910
Jul 17 03:08:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:08.045Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_usage_bytes\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda2\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545216106
Jul 17 03:08:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:08.100Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-ttyS2.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689545213582
Jul 17 03:08:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:08.134Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Pkt_Inbound\", app_name=\"sapp4\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", vsysId=\"1\"}" timestamp=1689545217817
Jul 17 03:08:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:08.140Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", phase=\"Unknown\", pod=\"prometheus-server-6ff784d758-76b74\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"5ee8d190-8de7-4d53-a103-d0969342cef5\"}" timestamp=1689545221227
Jul 17 03:08:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:08.159Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"overlay\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-9b07bfd510c4a8a6d3044d5a96fdba1ac94bb7d3878a68b4f6fe473de287479b-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689545213582
Jul 17 03:08:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:08.176Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"tmpfs\", name=\"run-k3s-containerd-io.containerd.grpc.v1.cri-sandboxes-3bb6529bd096436f633921ff850e4b450d60b63e799552efd5000ac08a2646d1-shm.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689545213582
Jul 17 03:08:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:08.193Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_memory_Hugepagesize_bytes\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545223910
Jul 17 03:08:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:08.216Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db140\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545218290
Jul 17 03:08:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:08.348Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2did-scsi\\\\x2d362cea7f085ed36002c25b4d2a14582f7\\\\x2dpart2.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689545213582
Jul 17 03:08:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:08.393Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_info\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_endpoint=\"prometheus-pushgateway\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545221227
Jul 17 03:08:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:08.408Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_persistentvolume_status_phase\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", persistentvolume=\"pvc-33f6b149-604d-41ff-a45b-c68cab677f47\", phase=\"Failed\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545221227
Jul 17 03:08:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:08.451Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_cpu_seconds_total\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-podcd272aa8_aacf_4f0d_971f_5dbf27e21826.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"slice\"}" timestamp=1689545213582
Jul 17 03:08:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:08.466Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", fstype=\"ext4\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/data/var/lib/kubelet/pods/cd272aa8-aacf-4f0d-971f-5dbf27e21826/volume-subpaths/telegraf/telegraf-security/0\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545223910
Jul 17 03:08:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:08.590Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-virtual-net-veth73a33e2a.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689545213582
Jul 17 03:08:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:08.612Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cpu=\"96\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545223910
Jul 17 03:08:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:11.350Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_STATS_metric_putBatchTime_90th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545230147
Jul 17 03:08:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:11.420Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_tsg_galaxy_table_gtpc_knowledge_base_metric_scanSize_95th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545230147
Jul 17 03:08:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:11.434Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_flushesQueuedCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"707c60fcbf33b487dea94876da6b9b4d\", table=\"default\", type=\"data\"}" timestamp=1689545230147
Jul 17 03:08:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:11.449Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_193_1_metric_flushMemstoreSize_25th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545230147
Jul 17 03:08:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:11.598Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_194_3_metric_getTime_25th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545230147
Jul 17 03:08:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:11.654Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_1_metric_compactionOutputFileCount_num_ops\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545230147
Jul 17 03:08:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:11.671Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_compactionsFailedCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", namespace=\"SYSTEM\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"9dc49d632110b5e10155ec645fe51a62\", table=\"TASK\", type=\"data\"}" timestamp=1689545230147
Jul 17 03:08:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:11.719Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_2_metric_putTime_99_9th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545230147
Jul 17 03:08:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:11.747Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_scanCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"56761702d12eb0aa3a225c8b4de1a8a6\", table=\"proxy_hos_bucket\", type=\"data\"}" timestamp=1689545230147
Jul 17 03:08:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:11.853Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_getCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"rtp_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"c95c3010cb97021df6c788a2c303ba49\", table=\"index_partfile_rtp_hos_bucket\", type=\"index\"}" timestamp=1689545230147
Jul 17 03:08:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:11.962Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_3_metric_scanTime_95th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545230147
Jul 17 03:08:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:11.980Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"java_lang_Threading_SynchronizerUsageSupported\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545230147
Jul 17 03:08:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:11.997Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_193_1_metric_majorCompactionInputSize_99th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545230147
Jul 17 03:08:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:12.096Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_session_record_hos_bucket_metric_compactionOutputSize_99_9th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545230147
Jul 17 03:08:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:12.111Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_compactionsFailedCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"rtp_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"f7f1f4fc67718d48d80d0634e0df3169\", table=\"index_time_rtp_hos_bucket\", type=\"index\"}" timestamp=1689545230147
Jul 17 03:08:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:12.133Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_default_metric_getTime_mean\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545230147
Jul 17 03:08:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:12.284Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_MUTEX_metric_minStoreFileAge\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545230147
Jul 17 03:08:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:12.593Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_lastMajorCompactionAge\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"proxy_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"7bcf63599704330b114c0c7e0332621c\", table=\"index_time_proxy_hos_bucket\", type=\"index\"}" timestamp=1689545230147
Jul 17 03:08:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:12.638Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_maxCompactionQueueSize\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"96a55ea6420b424b55926f18a013afc6\", table=\"firewall_hos_bucket\", type=\"data\"}" timestamp=1689545230147
Jul 17 03:08:13 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:13.131Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_frequency_max_hertz\", asset=\"twa-dellr740-dt002\", asset_id=\"39\", cpu=\"46\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"OLAP-Node-Exporter-twa-dellr740-dt002\", endpoint_id=\"259\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545232055
Jul 17 03:08:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:15.429Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_link_error_recovery_total\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_10\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545221230
Jul 17 03:08:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:15.446Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_multicast_packets_transmitted_total\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_1\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545221230
Jul 17 03:08:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:15.521Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-netns-cni\\\\x2d180d281c\\\\x2d1eb7\\\\x2da247\\\\x2d2365\\\\x2dbea4accb9a54.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689545227046
Jul 17 03:08:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:15.539Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_created\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-1-firewall-96db44557\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545228076
Jul 17 03:08:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:15.560Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"13\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545221230
Jul 17 03:08:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:15.575Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"machine_scrape_error\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545229150
Jul 17 03:08:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:15.587Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"twa-tsgx07\", asset_id=\"103\", container=\"redis\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"cadvisor\", failure_type=\"pgmajfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"hierarchy\"}" timestamp=1689545213654
Jul 17 03:08:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:15.590Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", fstype=\"ext4\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/var/lib/kubelet/pods/64082a69-3be2-4b5e-9e3a-c0da16576d9e/volume-subpaths/telegraf/telegraf-security/0\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545221230
Jul 17 03:08:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:15.593Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_observed_generation\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"prometheus-alertmanager\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545228076
Jul 17 03:08:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:15.615Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_info\", asset=\"twa-tsgx07\", asset_id=\"103\", created_by_kind=\"ReplicaSet\", created_by_name=\"systemd-exporter-7f549d9485\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", host_ip=\"10.10.30.167\", host_network=\"true\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", node=\"localhost\", nz_agent_id=\"3\", pod=\"systemd-exporter-7f549d9485-jkp9n\", pod_ip=\"10.10.30.167\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"c435f080-d1a5-46a3-868f-b5d8209ce2ea\"}" timestamp=1689545228076
Jul 17 03:08:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:15.628Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-systemd-exporter-2gkbr\", project=\"TSG-OS\", project_id=\"7\", reason=\"NodeAffinity\", service=\"prometheus-kube-state-metrics\", uid=\"bb339da1-4895-434d-8144-1e7c35d0b13e\"}" timestamp=1689545228076
Jul 17 03:08:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:15.715Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"108\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545221230
Jul 17 03:08:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:15.722Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_sockets\", asset=\"twa-tsgx07\", asset_id=\"103\", container=\"kubernetes-dashboard\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545220085
Jul 17 03:08:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:15.733Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_replicas\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-1-firewall-574b9bd44b\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545228076
Jul 17 03:08:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:15.748Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"117\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545221230
Jul 17 03:08:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:15.770Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_free_bytes\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"shm\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/47d366d6ef4b664104cf40db6f60a2166269b11667be7a26e7d6459e697f6016/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545221230
Jul 17 03:08:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:15.801Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_up\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens4f2np2\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545221230
Jul 17 03:08:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:16.472Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_usage_bytes\", asset=\"twa-tsgx07\", asset_id=\"103\", container=\"coredns\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545218398
Jul 17 03:08:17 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:17.037Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_spec_strategy_rollingupdate_max_surge\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"local-path-provisioner\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545228076
Jul 17 03:08:17 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:17.084Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_memory_FilePmdMapped_bytes\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545221230
Jul 17 03:08:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:20.075Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_metadata_resource_version\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"chart-values-tsg-traffic-engine-numa-1\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545233949
Jul 17 03:08:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:20.364Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"102\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545228263
Jul 17 03:08:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:20.428Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"blk-availability.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"service\"}" timestamp=1689545230657
Jul 17 03:08:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:20.449Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dm-event.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"service\"}" timestamp=1689545230657
Jul 17 03:08:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:20.452Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_scrape_collector_duration_seconds\", asset=\"twa-tsgx06\", asset_id=\"101\", collector=\"tapestats\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545237612
Jul 17 03:08:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:20.471Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"77\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545228263
Jul 17 03:08:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:20.725Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"117\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545237612
Jul 17 03:08:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:20.752Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_restarts_total\", asset=\"twa-tsgx06\", asset_id=\"101\", container=\"helm\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-configmap-reloader-pmpzk\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"86d10a1f-8af3-4a7c-b9f6-a63e3e9ca659\"}" timestamp=1689545233949
Jul 17 03:08:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:20.752Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db15\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545231998
Jul 17 03:08:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:20.774Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Ipv4_Bit\", app_name=\"sapp4\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", vsysId=\"1\"}" timestamp=1689545233544
Jul 17 03:08:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:20.796Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_reads_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", container=\"redis\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545232024
Jul 17 03:08:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:20.803Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-3dc3f6f644490386c4e265c30b79b73ec4f3ad268db1bf16fe3de1b6ab2d17e2-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689545224356
Jul 17 03:08:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:20.809Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_errs_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens3f1np1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545228263
Jul 17 03:08:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:20.901Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", container=\"coredns\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Sync\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545227536
Jul 17 03:08:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:20.920Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_owner\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", job_name=\"helm-install-mrzcpd-exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", owner_is_controller=\"<none>\", owner_kind=\"<none>\", owner_name=\"<none>\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545233828
Jul 17 03:08:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:20.976Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_created\", asset=\"pcap-tsgx10\", asset_id=\"89\", configmap=\"chart-content-traefik\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545233828
Jul 17 03:08:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:20.999Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"twa-tsgx06\", asset_id=\"101\", condition=\"unknown\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", pod=\"tsg-traffic-engine-numa-0-firewall-5db5dbc44c-866zl\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"9145d365-24c7-4ff1-a1ee-0a800b33502b\"}" timestamp=1689545233949
Jul 17 03:08:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:21.026Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_annotations\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.mrzcpd-exporter.v1\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545233949
Jul 17 03:08:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:21.033Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"17\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545228263
Jul 17 03:08:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:21.053Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_device_id\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"br_dign_c\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545228263
Jul 17 03:08:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:21.075Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_frame_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"vethdf3e6cb0\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545228263
Jul 17 03:08:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:21.108Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_annotations\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"dashboard-metrics-scraper\", uid=\"325dd97e-7163-4029-ab6c-2360cae9de0a\"}" timestamp=1689545233949
Jul 17 03:08:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:21.165Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_bytes_total\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethef8c731a\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545237612
Jul 17 03:08:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:21.199Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_drop_total\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth52173962\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545237612
Jul 17 03:08:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:21.205Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_created\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", job_name=\"helm-install-mrzcpd-exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545233828
Jul 17 03:08:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:21.225Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_fifo_total\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth52173962\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545237612
Jul 17 03:08:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:21.227Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"configmap-reloader-6c55bbc49f-sdx4c\", project=\"TSG-OS\", project_id=\"7\", reason=\"NodeAffinity\", service=\"prometheus-kube-state-metrics\", uid=\"d69b2139-72e0-47ab-b29f-fa787b9e63b9\"}" timestamp=1689545233828
Jul 17 03:08:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:21.245Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"vethd6f33409\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545228263
Jul 17 03:08:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:21.266Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx06\", asset_id=\"101\", container=\"dashboard-metrics-scraper\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Total\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545227364
Jul 17 03:08:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:21.284Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db16\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545234840
Jul 17 03:08:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:21.290Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"70\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545237612
Jul 17 03:08:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:21.310Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_mtu_bytes\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"bond0\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545237612
Jul 17 03:08:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:21.330Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_virtual_memory_bytes\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"tsg-clixon-backend.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689545224356
Jul 17 03:08:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:21.331Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Udp_Offload_STM\", app_name=\"sapp4\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", vsysId=\"1\"}" timestamp=1689545233544
Jul 17 03:08:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:21.340Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"tsg_master_tamper\", app_name=\"tsg_master\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"traffic-engine\", line_name=\"fail\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", table_name=\"flow\", vsysId=\"1\"}" timestamp=1689545238534
Jul 17 03:08:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:21.406Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db241\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545231998
Jul 17 03:08:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:21.454Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_max_fds\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"gssproxy.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689545224356
Jul 17 03:08:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:21.632Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_processes\", asset=\"pcap-tsgx10\", asset_id=\"89\", container=\"telegraf-security\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689545231133
Jul 17 03:08:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:21.634Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_address_assign_type\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens8f0np0\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545228263
Jul 17 03:08:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:21.649Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_annotations\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_endpoint=\"ipmi-exporter\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545233828
Jul 17 03:08:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:21.723Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"50\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545228263
Jul 17 03:08:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:21.741Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_annotations\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", pod=\"dashboard-metrics-scraper-5b94897dc9-952r6\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"4b0459ae-aedd-4c07-81f2-0240bc0fb208\"}" timestamp=1689545233828
Jul 17 03:08:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:21.770Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_id\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"virtio_dign_s\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545228263
Jul 17 03:08:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:21.794Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_rss\", asset=\"pcap-tsgx10\", asset_id=\"89\", container=\"metrics-server\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545232655
Jul 17 03:08:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:21.807Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_metadata_generation\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-1-firewall-5d7bfbbbfb\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545233828
Jul 17 03:08:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:21.824Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_carrier_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth4df73582\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545228263
Jul 17 03:08:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:21.898Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Unknown\", pod=\"svclb-traefik-742ef4d3-452pc\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"88ae83c1-0ebf-40d5-bed7-6f1b41109c68\"}" timestamp=1689545233828
Jul 17 03:08:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:21.911Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_packets_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth301b7957\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545228263
Jul 17 03:08:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:21.969Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_packets_dropped_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"cadvisor\", interface=\"br_dign_c\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545229922
Jul 17 03:08:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:21.986Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_info\", asset=\"pcap-tsgx10\", asset_id=\"89\", configmap=\"kube-apiserver-legacy-service-account-token-tracking\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545233828
Jul 17 03:08:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:22.321Z caller=dedupe.go:112 component=remote level=error remote_name=52db6f url=http://10.10.10.159:10090/nz-agent/prometheus/proxy/api/v1/write msg="non-recoverable error" count=1000 exemplarCount=0 err="server returned HTTP status 400 : out of order sample"
Jul 17 03:08:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:23.948Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_system_seconds_total\", asset=\"msh-tsgx09\", asset_id=\"55\", container=\"telegraf-security\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689545235355
Jul 17 03:08:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:23.971Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx09\", asset_id=\"55\", cpu=\"28\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545234790
Jul 17 03:08:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:24.119Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_start_time\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-server-6ff784d758-wz5rs\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"ab6b1591-cfaa-4964-8fcc-a861d0862792\"}" timestamp=1689545231122
Jul 17 03:08:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:24.151Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_start_time\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"traefik-64f55bb67d-xrfgg\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"fd6dc4c2-5a78-40f9-8373-71b885a791c5\"}" timestamp=1689545231122
Jul 17 03:08:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:24.163Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_virtual_memory_bytes\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"mrzcpd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689545231102
Jul 17 03:08:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:24.173Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"tmpfs\", name=\"data-var-lib-kubelet-pods-89567228\\\\x2d6ae4\\\\x2d4ed5\\\\x2d8117\\\\x2d7627ccbebb6d-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2dbhmtx.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689545231102
Jul 17 03:08:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:24.251Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"gssproxy.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"forking\", type=\"service\"}" timestamp=1689545231102
Jul 17 03:08:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:24.278Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"rpcbind.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"service\"}" timestamp=1689545231102
Jul 17 03:08:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:24.293Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_annotations\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-configmap-reloader-84s59\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"47ba69e1-fd09-4e39-941a-89000685484e\"}" timestamp=1689545240068
Jul 17 03:08:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:24.298Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_dormant\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth0c6b6d72\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545234790
Jul 17 03:08:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:24.322Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Tcp_Link_Double\", app_name=\"sapp4\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", vsysId=\"1\"}" timestamp=1689545240446
Jul 17 03:08:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:24.340Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_virtual_memory_bytes\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"udisks2.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689545238416
Jul 17 03:08:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:24.365Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_swap_limit_bytes\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"kube-multus\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545243419
Jul 17 03:08:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:24.381Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545240160
Jul 17 03:08:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:24.386Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-7035b848e997cd4d97f011a1bf33ed4b4515a52dd45504069f0be79def55d97a-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689545231102
Jul 17 03:08:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:24.409Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_ulimits_soft\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"lb-tcp-443\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", ulimit=\"max_open_files\"}" timestamp=1689545230068
Jul 17 03:08:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:24.427Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas_unavailable\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"metrics-server\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545231122
Jul 17 03:08:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:24.445Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_metadata_resource_version\", asset=\"pcap-tsgx12\", asset_id=\"117\", configmap=\"kube-root-ca.crt\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545231122
Jul 17 03:08:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:24.514Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_labels\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"traefik-64f55bb67d\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545231122
Jul 17 03:08:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:24.579Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx09\", asset_id=\"55\", cpu=\"39\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545234790
Jul 17 03:08:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:24.598Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_errs_total\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens4f2np2\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545234790
Jul 17 03:08:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:24.646Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-3edc99ea9fb8cf294828bd9d36157ab408f7c2a6d759b4c3d9f6e34cd6e06acd-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689545231102
Jul 17 03:08:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:24.678Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_commands_failed_calls_total\", asset=\"msh-tsgx09\", asset_id=\"55\", cmd=\"zadd\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545237019
Jul 17 03:08:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:24.763Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-delete-tsg-traffic-engine-numa-1-bjvt6\", project=\"TSG-OS\", project_id=\"7\", reason=\"UnexpectedAdmissionError\", service=\"prometheus-kube-state-metrics\", uid=\"55e15df4-2d3f-4878-94dc-5fef3e71fbdb\"}" timestamp=1689545240068
Jul 17 03:08:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:24.779Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"msh-tsgx09\", asset_id=\"55\", cpu=\"44\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545234790
Jul 17 03:08:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:24.793Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_node_labels\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", node=\"localhost\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545231122
Jul 17 03:08:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:24.835Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_terminated\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"helm\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-traefik-nwxfj\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"229a6300-c79e-4be5-8127-0eafbc465dae\"}" timestamp=1689545231122
Jul 17 03:08:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:24.843Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2dpartuuid-199f9912\\\\x2d9614\\\\x2d45ed\\\\x2d8f7e\\\\x2dfb4a9cd1cf26.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689545238416
Jul 17 03:08:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:24.983Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"dign-client-lt9hj\", project=\"TSG-OS\", project_id=\"7\", reason=\"NodeAffinity\", service=\"prometheus-kube-state-metrics\", uid=\"a7373468-7e47-4929-8ff7-6d32b07661f2\"}" timestamp=1689545231122
Jul 17 03:08:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:24.995Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"76\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545240160
Jul 17 03:08:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:25.002Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failcnt\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"metrics-server\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545236107
Jul 17 03:08:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:25.008Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"96\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545240160
Jul 17 03:08:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:25.120Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"proc-sys-fs-binfmt_misc.automount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"automount\"}" timestamp=1689545231102
Jul 17 03:08:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:25.315Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"coredns\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Write\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545239832
Jul 17 03:08:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:25.329Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_owner\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", job_name=\"helm-install-configmap-reloader\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", owner_is_controller=\"<none>\", owner_kind=\"<none>\", owner_name=\"<none>\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545231122
Jul 17 03:08:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:25.397Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"127\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545240160
Jul 17 03:08:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:25.988Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_link_error_recovery_total\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545235810
Jul 17 03:08:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:26.042Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_physical_state_id\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_2\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545235810
Jul 17 03:08:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:26.057Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_changes_total\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethf2a1e699\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545235810
Jul 17 03:08:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:26.060Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_compressed_total\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"eno2\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545235810
Jul 17 03:08:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:26.089Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"21\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545235810
Jul 17 03:08:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:26.118Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_device_id\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens4f0np0\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545235810
Jul 17 03:08:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:26.291Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_owner\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", owner_is_controller=\"true\", owner_kind=\"Job\", owner_name=\"helm-install-mrzcpd-exporter\", pod=\"helm-install-mrzcpd-exporter-pns2w\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"524fda03-478b-44e7-9ee9-f14d8b203a6f\"}" timestamp=1689545238744
Jul 17 03:08:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:26.329Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"twa-tsgx09\", asset_id=\"107\", condition=\"true\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-pushgateway-bd476bcf9-zqlv8\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"880aa12b-51b0-4413-88cb-d06b2fc6d0c3\"}" timestamp=1689545238744
Jul 17 03:08:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:26.783Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_bytes_total\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"virtio_dign_c\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545235810
Jul 17 03:08:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:26.803Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_errs_total\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vetha8a8b4ed\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545235810
Jul 17 03:08:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:26.816Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"iscsi.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689545233907
Jul 17 03:08:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:26.872Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_annotations\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.tsg-traffic-engine-numa-0.v1\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545238744
Jul 17 03:08:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:26.905Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"85\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545235810
Jul 17 03:08:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:26.922Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_colls_total\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth45cfb0fe\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545235810
Jul 17 03:08:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:26.947Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2dlabel-ONIE\\\\x2dBOOT.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689545233907
Jul 17 03:08:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:27.034Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"twa-tsgx09\", asset_id=\"107\", container=\"firewall\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", state=\"sleeping\"}" timestamp=1689545242940
Jul 17 03:08:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:27.087Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db47\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545243549
Jul 17 03:08:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:27.108Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db47\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545243549
Jul 17 03:08:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:27.149Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_resident_memory_bytes\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"chronyd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689545233907
Jul 17 03:08:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:27.266Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-74cda326eb92d459f3a4a925081192c3a5d64bc29394fa6753ccc96e12d37e48.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"scope\"}" timestamp=1689545233907
Jul 17 03:08:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:27.323Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"scrape_samples_scraped\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_instance=\"localhost\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545245490
Jul 17 03:08:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:27.392Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"8\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545235810
Jul 17 03:08:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:27.410Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db202\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545243549
Jul 17 03:08:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:27.491Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_io_current\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"overlay_0-47\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545234953
Jul 17 03:08:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:27.543Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"exporter-systemd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"service\"}" timestamp=1689545233907
Jul 17 03:08:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:27.557Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"svclb-traefik-dbdae63f-svgf7\", project=\"TSG-OS\", project_id=\"7\", reason=\"UnexpectedAdmissionError\", service=\"prometheus-kube-state-metrics\", uid=\"b7d50d7a-bdb1-4746-8ca3-4cae9f1a4c87\"}" timestamp=1689545235682
Jul 17 03:08:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:27.572Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"coredns\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Sync\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545240787
Jul 17 03:08:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:27.575Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_multicast_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth6d9f498e\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545242882
Jul 17 03:08:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:27.615Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"127\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545235810
Jul 17 03:08:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:27.617Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda5\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", fstype=\"ext4\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/data/var/lib/kubelet/pods/0b0292d1-7c6a-4fad-a1f9-f14e3c7cb4f8/volume-subpaths/config-volume/telegraf-security/1\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545242882
Jul 17 03:08:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:27.662Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_scrape_collector_success\", asset=\"twa-tsgx09\", asset_id=\"107\", collector=\"rapl\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545235810
Jul 17 03:08:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:27.715Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:20-0000:20:03.1-0000:21:00.1-infiniband_mad-umad3.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689545233907
Jul 17 03:08:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:27.745Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"tsg_master_T_drop_log\", app_name=\"tsg_master\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"traffic-engine\", line_name=\"INTERIM-SESSION-RECORD\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", table_name=\"sendlog\", vsysId=\"1\"}" timestamp=1689545231010
Jul 17 03:08:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:27.800Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled_time\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"coredns-6dcd74846d-hdznl\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"270c9661-7d2c-4d16-8b15-3c2fb0ea9cf1\"}" timestamp=1689545238744
Jul 17 03:08:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:27.949Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"50\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545242882
Jul 17 03:08:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:28.254Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_carrier_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth20809015\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545242882
Jul 17 03:08:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:28.273Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-05a8d67c\\\\x2de611\\\\x2d49c0\\\\x2d83d9\\\\x2dd8b010e1bac4-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2d6lj75.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689545234114
Jul 17 03:08:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:28.498Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_compressed_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens6f1np1\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545242882
Jul 17 03:08:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:28.706Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"cortex_kv_request_duration_seconds_bucket\", agent_name=\"msh-kvm-nz001\", application=\"loki\", datacenter=\"MSH\", instance=\"127.0.0.1:13100\", kv_name=\"ingester-ring\", le=\"1.0\", operation=\"GET\", role=\"primary\", status_code=\"200\", type=\"memberlist\"}" timestamp=1689545248332
Jul 17 03:08:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:28.723Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"import-state.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"service\"}" timestamp=1689545234114
Jul 17 03:08:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:31.114Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_sockstat_UDPLITE6_inuse\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545240939
Jul 17 03:08:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:31.613Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_address_available\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_endpoint=\"ipmi-exporter\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545250436
Jul 17 03:08:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:31.646Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-76e087a56cab779cf96a899874139822324870dc5f2cec4e3ac4557ffdaadea5-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689545246907
Jul 17 03:08:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:31.670Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-76e087a56cab779cf96a899874139822324870dc5f2cec4e3ac4557ffdaadea5.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"scope\"}" timestamp=1689545246907
Jul 17 03:08:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:31.764Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"46\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545240939
Jul 17 03:08:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:31.767Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_fully_labeled_replicas\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"configmap-reloader-6c55bbc49f\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545239276
Jul 17 03:08:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:31.823Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"debugfs\", name=\"sys-kernel-debug.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689545241015
Jul 17 03:08:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:31.859Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db235\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545246438
Jul 17 03:08:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:31.866Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_file_descriptors\", asset=\"msh-tsgx05\", asset_id=\"67\", container=\"kube-multus\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545232388
Jul 17 03:08:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:31.881Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"92\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545240939
Jul 17 03:08:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:31.903Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"watchdog.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689545246907
Jul 17 03:08:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:32.220Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_drop_total\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens7f1np1\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545239093
Jul 17 03:08:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:32.231Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_errs_total\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens7f1np1\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545239093
Jul 17 03:08:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:32.304Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"overlay\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-1d68b5d27a8f4e5d7f1cfba0571d0ccc472629c36a67933adc44728f3233a8d2-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689545242644
Jul 17 03:08:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:32.325Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_bytes_total\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens8f0np0\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545239093
Jul 17 03:08:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:32.377Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_state_started\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"dashboard-metrics-scraper\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", pod=\"dashboard-metrics-scraper-5b94897dc9-bljqz\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"177d5113-bd61-4a37-9476-be6e91ea55a1\"}" timestamp=1689545239276
Jul 17 03:08:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:32.393Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"tmpfs\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/var/lib/kubelet/pods/b7e44624-9b20-4189-9ac3-b8a15c1776a6/volumes/kubernetes.io~projected/kube-api-access-mwm7p\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545240791
Jul 17 03:08:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:32.400Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_reads_total\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/sys/fs/cgroup\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545240232
Jul 17 03:08:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:32.418Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_annotations\", asset=\"twa-tsgx03\", asset_id=\"95\", configmap=\"coredns\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545250436
Jul 17 03:08:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:32.434Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.grpc.v1.cri-sandboxes-690513d2a1680c29bf9531ad9671a97d4211fb1b16f830d7103de3850fc686f8-shm.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689545246907
Jul 17 03:08:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:32.530Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"52\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545240605
Jul 17 03:08:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:32.537Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Unknown\", pod=\"helm-install-ipmi-exporter-h7mdw\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"14488225-fd95-4f38-a9af-d4af570ca0f1\"}" timestamp=1689545250436
Jul 17 03:08:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:32.554Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_address_assign_type\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens4f1np1\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545240605
Jul 17 03:08:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:32.581Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_packets_total\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens1f0np0\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545240605
Jul 17 03:08:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:32.594Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx05\", asset_id=\"99\", container=\"firewall\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Read\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689545234875
Jul 17 03:08:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:32.609Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"27\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545240791
Jul 17 03:08:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:32.679Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", phase=\"Pending\", pod=\"tsg-traffic-engine-numa-1-firewall-6dd49d747-slxl4\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"bdd10b0d-cc33-4a4f-9622-89ebf267e6c3\"}" timestamp=1689545239898
Jul 17 03:08:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:32.700Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_labels\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_endpoint=\"cm-policy-local-cache-redis-metrics\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545239898
Jul 17 03:08:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:33.033Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_info\", address=\"ee:75:02:e0:21:df\", asset=\"twa-tsgx05\", asset_id=\"99\", broadcast=\"ff:ff:ff:ff:ff:ff\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth17383320\", duplex=\"full\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operstate=\"up\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545240605
Jul 17 03:08:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:33.058Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_queue_length\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens7f1np1\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545240605
Jul 17 03:08:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:33.105Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545240605
Jul 17 03:08:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:33.142Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_oom_events_total\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"monit-stream\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545233619
Jul 17 03:08:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:33.148Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_scrape_collector_success\", asset=\"twa-tsgx05\", asset_id=\"99\", collector=\"fibrechannel\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545240605
Jul 17 03:08:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:33.204Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_hwmon_temp_celsius\", asset=\"twa-tsgx05\", asset_id=\"99\", chip=\"pci0000:00_0000:00:18_3\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", sensor=\"temp6\", service=\"prometheus-node-exporter\"}" timestamp=1689545240605
Jul 17 03:08:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:33.230Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_buck_hash_sys_bytes\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"ipmi-exporter\"}" timestamp=1689545243498
Jul 17 03:08:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:33.281Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_disk_io_now\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"sda\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545240605
Jul 17 03:08:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:33.650Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_colls_total\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"br_dign_s\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545240605
Jul 17 03:08:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:33.957Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_running\", asset=\"twa-tsgx05\", asset_id=\"99\", container=\"ipmi-exporter\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"ipmi-exporter-6f8464dd94-lvhsz\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"a0e5cbc1-daaa-4b9e-8b09-4f9b6a34498e\"}" timestamp=1689545239898
Jul 17 03:08:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:33.983Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_labels\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-traefik-crd-bf9k4\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"39ad39af-b9c7-4eb1-a7db-349fcd19b742\"}" timestamp=1689545239898
Jul 17 03:08:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:33.994Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"41\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545240605
Jul 17 03:08:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:34.007Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_compressed_total\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth989c1124\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545240605
Jul 17 03:08:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:34.014Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_scrape_collector_success\", asset=\"twa-tsgx05\", asset_id=\"99\", collector=\"zfs\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545240605
Jul 17 03:08:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:34.031Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db93\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545249260
Jul 17 03:08:36 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:36.808Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kafka_server_BrokerTopicMetrics_OneMinuteRate\", asset=\"pcap-dellr740-dt002\", asset_id=\"33\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-Kafka-pcap-dellr740-dt002\", endpoint_id=\"101\", module=\"DC-Kafka\", module_id=\"5\", name=\"TotalProduceRequestsPerSec\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", topic=\"VOIP-CONVERSATION-RECORD\"}" timestamp=1689545255787
Jul 17 03:08:39 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:39.065Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-dellr740-dt003\", asset_id=\"41\", cpu=\"8\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"OLAP-Node-Exporter-twa-dellr740-dt003\", endpoint_id=\"261\", mode=\"nice\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"hbase_master,clickhouse_data,kafka,gohangout,zookeeper,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545258476
Jul 17 03:08:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:40.293Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_scaling_frequency_max_hertz\", asset=\"twa-dellr740-dt003\", asset_id=\"41\", cpu=\"22\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"OLAP-Node-Exporter-twa-dellr740-dt003\", endpoint_id=\"261\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"hbase_master,clickhouse_data,kafka,gohangout,zookeeper,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545258476
Jul 17 03:08:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:40.473Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"9\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545251647
Jul 17 03:08:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:40.500Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"twa-tsgx04\", asset_id=\"97\", condition=\"true\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-systemd-exporter-rhvzp\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"4283428c-b6e9-48f3-9c81-57aafbe8e47a\"}" timestamp=1689545260034
Jul 17 03:08:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:40.516Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_time_clocksource_available_info\", asset=\"twa-tsgx04\", asset_id=\"97\", clocksource=\"acpi_pm\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"0\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545251647
Jul 17 03:08:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:40.518Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled_time\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", pod=\"kubernetes-dashboard-66dd5b456-7f4bj\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"866c527e-d063-4cfb-b039-fa190c69b128\"}" timestamp=1689545260034
Jul 17 03:08:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:40.536Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db34\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545253827
Jul 17 03:08:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:40.538Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"126\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545251647
Jul 17 03:08:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:40.607Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_threads_max\", asset=\"twa-tsgx04\", asset_id=\"97\", container=\"firewall\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689545236166
Jul 17 03:08:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:40.639Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_info\", asset=\"twa-tsgx04\", asset_id=\"97\", cluster_ip=\"10.43.36.235\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"dashboard-metrics-scraper\", uid=\"ff15c352-595d-4ee7-9f33-949732c6163e\"}" timestamp=1689545260034
Jul 17 03:08:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:40.673Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"84\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545251647
Jul 17 03:08:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:40.792Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"sleeping\"}" timestamp=1689545247766
Jul 17 03:08:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:40.867Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"hasp_monitor.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"simple\", type=\"service\"}" timestamp=1689545255764
Jul 17 03:08:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:40.963Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_bytes_total\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"cadvisor\", interface=\"ens7f0np0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545247766
Jul 17 03:08:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:41.003Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_packets_dropped_total\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"cadvisor\", interface=\"eno35np2\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545247766
Jul 17 03:08:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:41.023Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_packets_total\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens4f1np1\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545251647
Jul 17 03:08:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:41.034Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_numFilesCompactedCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"c797c1b183075738b860976f05c9f684\", table=\"knowledge_base_hos_bucket\", type=\"data\"}" timestamp=1689545259757
Jul 17 03:08:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:41.041Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"39\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545251647
Jul 17 03:08:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:41.060Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"getty-pre.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"target\"}" timestamp=1689545255764
Jul 17 03:08:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:41.060Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_memStoreSize\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"assessment_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"77d5f82e5f12961775c07304d24c8707\", table=\"index_partfile_assessment_hos_bucket\", type=\"index\"}" timestamp=1689545259757
Jul 17 03:08:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:41.074Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"java_lang_GarbageCollector_LastGcInfo_memoryUsageBeforeGc_init\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", key=\"Par Survivor Space\", module=\"DC-HBase\", module_id=\"3\", name=\"ParNew\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545259757
Jul 17 03:08:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:41.077Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"18\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545251647
Jul 17 03:08:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:41.093Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_10_194_1_metric_flushMemstoreSize_min\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545259757
Jul 17 03:08:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:41.095Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_10_193_1_metric_majorCompactionOutputSize_90th_percentile\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545259757
Jul 17 03:08:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:41.097Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_device_id\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth4e5d6c5d\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545251647
Jul 17 03:08:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:41.122Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_io_time_weighted_seconds_total\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545245701
Jul 17 03:08:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:41.210Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_filteredReadRequestCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"firewall_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"8a95c2468342cbbd723e30d658cb8faa\", table=\"index_time_firewall_hos_bucket\", type=\"index\"}" timestamp=1689545259757
Jul 17 03:08:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:41.600Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_filteredReadRequestCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"proxy_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"84e88cb916f2401f74e9ca992798ed07\", table=\"index_time_proxy_hos_bucket\", type=\"index\"}" timestamp=1689545259757
Jul 17 03:08:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:41.783Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_tsg_galaxy_table_relation_account_framedip_metric_appendTime_95th_percentile\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545259757
Jul 17 03:08:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:42.021Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_10_194_1_metric_splitTime_mean\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545259757
Jul 17 03:08:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:42.429Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", container=\"telegraf\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Read\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689545252657
Jul 17 03:08:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:42.554Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_annotations\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"traefik\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545255154
Jul 17 03:08:44 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:44.025Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_alloc_bytes_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545254838
Jul 17 03:08:44 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:44.059Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"72\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545256242
Jul 17 03:08:44 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:44.100Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", cpu=\"123\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545256242
Jul 17 03:08:44 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:44.165Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_multicast_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"vethb1a97a04\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545256242
Jul 17 03:08:44 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:44.183Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_errs_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth585262fb\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545256242
Jul 17 03:08:44 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:44.211Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_spec_parallelism\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", job_name=\"helm-install-cm-policy-local-cache\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545255154
Jul 17 03:08:44 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:44.249Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"tmpfs\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/var/lib/kubelet/pods/98daa3ea-45f0-41c2-be94-512fb3e2a016/volumes/kubernetes.io~projected/kube-api-access-7768b\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545256242
Jul 17 03:08:44 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:44.443Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled_time\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"coredns-79c6ccb74b-rpzl7\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"be245aed-e373-405f-8d79-93c5764d578b\"}" timestamp=1689545255154
Jul 17 03:08:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:49.043Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_194_2_metric_majorCompactionOutputFileCount_min\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545267439
Jul 17 03:08:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:49.205Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_storeCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"firewall_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"312be68cb14448f0189460056d4d500f\", table=\"index_time_firewall_hos_bucket\", type=\"index\"}" timestamp=1689545267439
Jul 17 03:08:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:49.236Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"java_lang_MemoryPool_UsageThresholdSupported\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"CMS Old Gen\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545267439
Jul 17 03:08:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:49.308Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_deleteCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"proxy_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"211911888371e8f4b47031fc3c87006b\", table=\"index_filename_proxy_hos_bucket\", type=\"index\"}" timestamp=1689545267439
Jul 17 03:08:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:49.342Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_compactionsFailedCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"4742e1c0c5331a4df4bdd23109c9d4a9\", table=\"troubleshooting_hos_bucket\", type=\"data\"}" timestamp=1689545267439
Jul 17 03:08:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:49.373Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_minStoreFileAge\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"rtp_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"8b4bb800fda3b83e73863db9941293cf\", table=\"index_partfile_rtp_hos_bucket\", type=\"index\"}" timestamp=1689545267439
Jul 17 03:08:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:49.430Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_session_record_hos_bucket_metric_compactionInputFileCount_min\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545267439
Jul 17 03:08:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:49.525Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_packets_total\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"eno34np1\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545261797
Jul 17 03:08:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:49.555Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_scrape_collector_duration_seconds\", asset=\"twa-tsgx10\", asset_id=\"109\", collector=\"udp_queues\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545261797
Jul 17 03:08:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:49.646Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db185\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545266625
Jul 17 03:08:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:49.775Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_SEQUENCE_metric_scanSize_90th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545267439
Jul 17 03:08:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:49.828Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_193_1_metric_flushTime_75th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545267439
Jul 17 03:08:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:49.859Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_readRequestCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"c4c7ee714d7e524ae4636d78724f09fa\", table=\"rtp_hos_bucket\", type=\"data\"}" timestamp=1689545267439
Jul 17 03:08:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:49.964Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_replicas\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"prometheus-server-6ff784d758\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545261568
Jul 17 03:08:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:50.021Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_frequency_min_hertz\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"1\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545266709
Jul 17 03:08:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:50.116Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_194_4_metric_compactionTime_max\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545267439
Jul 17 03:08:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:50.167Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_address_assign_type\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens1f1np1\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545261797
Jul 17 03:08:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:50.187Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2did-scsi\\\\x2dSDELL_PERC_H345_Adp_00f5c79fdded05272c008ed5adf0a7ce\\\\x2dpart1.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689545261057
Jul 17 03:08:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:50.204Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_mod_info\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"github.com/prometheus/client_golang\", nz_agent_id=\"3\", program=\"systemd_exporter\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", version=\"v1.5.1\"}" timestamp=1689545261057
Jul 17 03:08:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:50.230Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_limit_bytes\", asset=\"twa-tsgx08\", asset_id=\"105\", container=\"firewall\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689545267561
Jul 17 03:08:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:50.284Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_time_zone_offset_seconds\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-olap003\", endpoint_id=\"235\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"yarn_resourcemanager,zookeeper,nacos,kafka,spark,druid_data,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\", time_zone=\"UTC\"}" timestamp=1689545269713
Jul 17 03:08:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:50.359Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_dos_table_ddos_traffic_baselines_metric_incrementTime_median\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545267439
Jul 17 03:08:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:50.392Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_1_metric_flushTime_95th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545267439
Jul 17 03:08:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:50.419Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_4_metric_majorCompactionInputFileCount_max\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545267439
Jul 17 03:08:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:50.457Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_scaling_frequency_max_hertz\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"62\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545266709
Jul 17 03:08:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:50.529Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_dormant\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"br_dign_s\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545266709
Jul 17 03:08:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:50.551Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"78\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545266709
Jul 17 03:08:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:50.608Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_reads_bytes_total\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"prometheus-node-exporter\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545257961
Jul 17 03:08:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:50.625Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_created\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"configmap-reloader\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545259668
Jul 17 03:08:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:50.666Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_disk_discarded_sectors_total\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"sda\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545261797
Jul 17 03:08:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:50.684Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.grpc.v1.cri-sandboxes-8f86fce574672ef609e05b8d638aa39ded7de6f7c3f405c92d51d2cea4e750d0-shm.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689545261057
Jul 17 03:08:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:50.772Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_usage_bytes\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/data/var/lib/kubelet/pods/32707ee1-6f45-4d2c-8e0e-78edc233adc3/volumes/kubernetes.io~projected/kube-api-access-x64jh\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545264873
Jul 17 03:08:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:50.781Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_vmstat_pgpgin\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-olap003\", endpoint_id=\"235\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"yarn_resourcemanager,zookeeper,nacos,kafka,spark,druid_data,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545269713
Jul 17 03:08:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:50.813Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2dpartuuid-cfede76c\\\\x2db822\\\\x2d4103\\\\x2d91fd\\\\x2ddb6deae3e600.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689545266554
Jul 17 03:08:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:50.833Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"twa-tsgx08\", asset_id=\"105\", container=\"kubernetes-dashboard\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", failure_type=\"pgfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"hierarchy\"}" timestamp=1689545257372
Jul 17 03:08:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:50.911Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"twa-tsgx08\", asset_id=\"105\", condition=\"Available\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"systemd-exporter\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"false\"}" timestamp=1689545261568
Jul 17 03:08:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:50.942Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_frequency_min_hertz\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"126\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545266709
Jul 17 03:08:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:50.957Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_receive_remote_physical_errors_total\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_0\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545261797
Jul 17 03:08:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:50.969Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_link_error_recovery_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_11\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545266709
Jul 17 03:08:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:50.974Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failcnt\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"lb-tcp-80\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545241805
Jul 17 03:08:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:50.982Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_multicast_total\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethbece5312\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545261797
Jul 17 03:08:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:51.002Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"twa-tsgx10\", asset_id=\"109\", cpu=\"63\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545261797
Jul 17 03:08:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:51.030Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_processes\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"kube-state-metrics\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545251782
Jul 17 03:08:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:51.049Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_last_terminated_reason\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"mrapm-device\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"mrzcpd-exporter-5cf7f967cc-mp8pv\", project=\"TSG-OS\", project_id=\"7\", reason=\"Unknown\", service=\"prometheus-kube-state-metrics\", uid=\"71f6269b-5154-46cd-b538-223c28b8f982\"}" timestamp=1689545259668
Jul 17 03:08:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:51.104Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_bytes_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", interface=\"ens2f1np1\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545266901
Jul 17 03:08:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:51.130Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_annotations\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\", uid=\"a2db026d-9179-4773-9df6-8ba7301d0478\"}" timestamp=1689545259668
Jul 17 03:08:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:51.144Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_rate_bytes_per_second\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_11\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545261797
Jul 17 03:08:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:51.156Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_colls_total\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth767e7158\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545261797
Jul 17 03:08:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:51.294Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx10\", asset_id=\"109\", cpu=\"74\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545261797
Jul 17 03:08:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:51.305Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_fifo_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth66a8f63b\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545266709
Jul 17 03:08:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:51.319Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:60-0000:60:03.1-0000:63:00.0-net-eno33np0.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689545266554
Jul 17 03:08:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:51.328Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_packets_transmitted_total\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_6\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545261797
Jul 17 03:08:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:51.369Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"twa-tsgx10\", asset_id=\"109\", cpu=\"83\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545261797
Jul 17 03:08:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:51.422Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_scrape_collector_duration_seconds\", asset=\"twa-tsgx10\", asset_id=\"109\", collector=\"nfsd\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545261797
Jul 17 03:08:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:51.438Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db163\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545266625
Jul 17 03:08:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:51.574Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"50\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545266709
Jul 17 03:08:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:52.518Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", cpu=\"9\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545269337
Jul 17 03:08:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:52.661Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_writes_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", container=\"coredns\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545269307
Jul 17 03:08:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:52.800Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_ulimits_soft\", asset=\"pcap-tsgx11\", asset_id=\"115\", container=\"lb-tcp-443\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", ulimit=\"max_open_files\"}" timestamp=1689545253331
Jul 17 03:08:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:52.819Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_completion_time\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-configmap-reloader-smwwp\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"26f82780-5515-438b-98f1-fdfb1a2c1042\"}" timestamp=1689545264478
Jul 17 03:08:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:53.309Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"msh-tsgx08\", asset_id=\"65\", cpu=\"97\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545268328
Jul 17 03:08:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:53.502Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_threads\", asset=\"msh-tsgx12\", asset_id=\"113\", container=\"configmap-reloader\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545254496
Jul 17 03:08:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:53.521Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_scrape_collector_success\", asset=\"msh-tsgx12\", asset_id=\"113\", collector=\"entropy\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545263391
Jul 17 03:08:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:53.619Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_dormant\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"bond0\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545268328
Jul 17 03:08:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:53.683Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_created\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", job_name=\"helm-install-ipmi-exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545264478
Jul 17 03:08:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:53.764Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"pcap-tsgx07\", asset_id=\"85\", condition=\"true\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-traefik-l4fwv\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"d32f8ba9-3fb3-4850-ae59-5587198ad4cc\"}" timestamp=1689545262058
Jul 17 03:08:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:53.779Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_reads_total\", asset=\"msh-tsgx12\", asset_id=\"113\", container=\"dign-server-web\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545251336
Jul 17 03:08:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:53.799Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_writes_bytes_total\", asset=\"msh-tsgx12\", asset_id=\"113\", container=\"exporter\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545257461
Jul 17 03:08:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:53.837Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_info\", asset=\"pcap-tsgx11\", asset_id=\"115\", configmap=\"cm-policy-local-cache-redis-configuration\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545264478
Jul 17 03:08:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:53.855Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_type\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"chart-values-systemd-exporter\", service=\"prometheus-kube-state-metrics\", type=\"Opaque\"}" timestamp=1689545262058
Jul 17 03:08:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:53.877Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", container=\"local-path-provisioner\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Async\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545258986
Jul 17 03:08:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:53.890Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"phy_tx_error_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", dev=\"ens7f1np1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"mrzcpd-exporter\"}" timestamp=1689545271521
Jul 17 03:08:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:53.916Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", cpu=\"116\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545269337
Jul 17 03:08:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:53.936Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_cpu_seconds_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-pod80d17ee2_8132_4555_a25c_5aba9a6428c8.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"slice\"}" timestamp=1689545260884
Jul 17 03:08:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:53.958Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-8b10ec3db74a68f5e84a9cb7a43dc3979390bf30068ab345db69ecd7e0c5f962.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"scope\"}" timestamp=1689545265496
Jul 17 03:08:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:53.996Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_terminated\", asset=\"pcap-tsgx11\", asset_id=\"115\", container=\"telegraf\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", pod=\"tsg-traffic-engine-numa-1-firewall-6f574c7957-cv5ql\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"05cea850-febe-4ad9-a01e-de2d8c0e8a8b\"}" timestamp=1689545264478
Jul 17 03:08:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:54.038Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_replicas\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-0-firewall-7cbb5bf99c\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545268950
Jul 17 03:08:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:54.063Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_annotations\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_endpoint=\"metrics-server\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545268950
Jul 17 03:08:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:54.087Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Running\", pod=\"helm-install-cm-policy-local-cache-5ktrg\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"6b46916c-eeef-4c1f-97c6-88043a8c1ae7\"}" timestamp=1689545268950
Jul 17 03:08:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:54.105Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_colls_total\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens2f1np1\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545268328
Jul 17 03:08:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:54.157Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_device_id\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth132a9a24\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545259409
Jul 17 03:08:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:54.179Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"multi-user.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"target\"}" timestamp=1689545260884
Jul 17 03:08:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:54.198Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cpu=\"114\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545259409
Jul 17 03:08:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:54.296Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"tmpfs\", name=\"data-var-lib-kubelet-pods-9c7f063f\\\\x2d9d02\\\\x2d46df\\\\x2d96ec\\\\x2d056a84a8d9d6-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2dpp86j.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689545265496
Jul 17 03:08:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:54.326Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:60-0000:60:03.1-0000:63:00.3-net-eno36np3.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689545272135
Jul 17 03:08:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:54.364Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_spec_strategy_rollingupdate_max_unavailable\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"prometheus-alertmanager\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545262058
Jul 17 03:08:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:54.391Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-pod21f91134_966a_424e_a89e_96c924c64f07.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"slice\"}" timestamp=1689545265496
Jul 17 03:08:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:54.401Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Failed\", pod=\"helm-install-cm-policy-local-cache-tpwbz\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"25f9128d-9e49-4c7b-b03c-90b4160244e3\"}" timestamp=1689545262058
Jul 17 03:08:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:54.421Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cpu=\"84\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545259409
Jul 17 03:08:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:54.431Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_protocol_type\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth7f2a971d\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545259409
Jul 17 03:08:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:54.444Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Tcp_Link_C2S_ALL\", app_name=\"sapp4\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", vsysId=\"1\"}" timestamp=1689545261765
Jul 17 03:08:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:54.460Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_ready_replicas\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"kubernetes-dashboard-66dd5b456\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545262058
Jul 17 03:08:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:54.490Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-netns-cni\\\\x2d8713bdc8\\\\x2d994b\\\\x2d97d4\\\\x2d6d60\\\\x2d0e557b87faa5.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689545260884
Jul 17 03:08:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:54.503Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_annotations\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"chart-values-tsg-traffic-engine-numa-1\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545268950
Jul 17 03:08:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:54.527Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:a0-0000:a0:03.1-0000:a1:00.1-infiniband_mad-umad7.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689545260884
Jul 17 03:08:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:54.543Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_cpu_seconds_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"slice\"}" timestamp=1689545260884
Jul 17 03:08:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:54.639Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-virtual-net-veth44c94df0.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689545260884
Jul 17 03:08:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:54.642Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", container=\"traefik\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"cadvisor\", failure_type=\"pgmajfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"container\"}" timestamp=1689545268597
Jul 17 03:08:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:54.763Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas_updated\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"dashboard-metrics-scraper\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545268950
Jul 17 03:08:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:54.784Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_created\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"cm-cache-extra-secrets\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545268950
Jul 17 03:08:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:54.804Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx08\", asset_id=\"65\", container=\"ipmi-exporter\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Total\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545247157
Jul 17 03:08:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:54.866Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-tsg-traffic-engine-numa-1-2fhs9\", project=\"TSG-OS\", project_id=\"7\", reason=\"UnexpectedAdmissionError\", service=\"prometheus-kube-state-metrics\", uid=\"2660dc42-c906-449c-9c29-3626178c08d7\"}" timestamp=1689545268663
Jul 17 03:08:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:54.884Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_complete\", asset=\"msh-tsgx08\", asset_id=\"65\", condition=\"true\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", job_name=\"helm-install-tsg-traffic-engine-numa-0\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545268950
Jul 17 03:08:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:54.888Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"3\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545263391
Jul 17 03:08:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:54.904Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", container=\"lb-tcp-443\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"cadvisor\", failure_type=\"pgfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"hierarchy\"}" timestamp=1689545264657
Jul 17 03:08:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:54.907Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_down_changes_total\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens2f0np0\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545263391
Jul 17 03:08:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:54.916Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_type\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"chart-values-tsg-traffic-engine-numa-1\", service=\"prometheus-kube-state-metrics\", type=\"Opaque\"}" timestamp=1689545268663
Jul 17 03:08:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:54.927Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-a40cbd61721b668d2576d07d27214c67b24578a512699142d048a7961fef51b8.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"scope\"}" timestamp=1689545271179
Jul 17 03:08:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:54.939Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-udev-trigger.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689545272135
Jul 17 03:08:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:54.987Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cpu=\"70\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545259409
Jul 17 03:08:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:54.995Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_repl_backlog_is_active\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545258659
Jul 17 03:08:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:55.004Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_queue_length\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens1f0np0\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545259409
Jul 17 03:08:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:55.010Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"abrt-journal-core.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"service\"}" timestamp=1689545271179
Jul 17 03:08:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:55.023Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-4832029edfe2512afd7c667f07ba975d766964db02fc0eb1e8d68fedfdb9eafa-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689545271179
Jul 17 03:08:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:55.105Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:00-0000:00:01.1-0000:01:00.0-infiniband-mlx5_0.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689545271179
Jul 17 03:08:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:55.122Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"var-lib-kubelet-pods-8b2e9710\\\\x2d4fd8\\\\x2d4c6f\\\\x2d8e5f\\\\x2d798d330e80a6-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2dq5fl9.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689545271179
Jul 17 03:08:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:56.370Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_working_set_bytes\", asset=\"pcap-tsgx03\", asset_id=\"73\", container=\"prometheus-alertmanager\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545267275
Jul 17 03:08:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:56.385Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_ips\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", ip=\"10.42.0.108\", ip_family=\"4\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"local-path-provisioner-957fdf8bc-qmcnx\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"43749d42-dc70-4f09-84cc-b9a556f85f71\"}" timestamp=1689545273389
Jul 17 03:08:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:56.434Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"42\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545265883
Jul 17 03:08:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:56.562Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", container=\"dign-server-dns\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Discard\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545256075
Jul 17 03:08:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:56.580Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas_updated\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"metrics-server\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545273389
Jul 17 03:08:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:56.927Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"37\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545265883
Jul 17 03:08:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:56.945Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"90\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545265883
Jul 17 03:08:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:57.010Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-5b5ea4ec\\\\x2d9b19\\\\x2d421d\\\\x2d88cb\\\\x2d9ffacec87f13-volume\\\\x2dsubpaths-config\\\\x2dvolume-telegraf-1.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689545263372
Jul 17 03:08:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:57.071Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_errs_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth19a516c9\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545265883
Jul 17 03:08:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:57.089Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"98\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545265883
Jul 17 03:08:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:57.213Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-pod43749d42_dc70_4f09_84cc_b9a556f85f71.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"slice\"}" timestamp=1689545263372
Jul 17 03:08:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:57.320Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"13\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545265883
Jul 17 03:08:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:57.336Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_disk_read_time_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"sda\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545265883
Jul 17 03:08:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:57.354Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_address_not_ready\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_endpoint=\"prometheus-kube-state-metrics\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545273389
Jul 17 03:08:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:57.420Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"67\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545265883
Jul 17 03:08:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:57.493Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"26\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545265883
Jul 17 03:08:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:57.633Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_up_changes_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens3f1np1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545265883
Jul 17 03:08:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:57.725Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"50\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545265883
Jul 17 03:08:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:57.746Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db65\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545266475
Jul 17 03:08:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:57.762Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Udp_Link_Del\", app_name=\"sapp4\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", vsysId=\"1\"}" timestamp=1689545270331
Jul 17 03:08:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:57.790Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_info\", address=\"00:00:00:00:00:00\", asset=\"msh-dellr740-dt003\", asset_id=\"9\", broadcast=\"00:00:00:00:00:00\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"lo\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-dt003\", endpoint_id=\"229\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"hbase_master,clickhouse_data,kafka,gohangout,zookeeper,hadoop_datanode,hbase_region,yarn_nodemanager\", operstate=\"unknown\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545277407
Jul 17 03:08:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:57.794Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2duuid-e839131a\\\\x2d9d2a\\\\x2d44ac\\\\x2da281\\\\x2d094287eb2eb8.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689545263372
Jul 17 03:08:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:57.810Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sshd-keygen.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"target\"}" timestamp=1689545263372
Jul 17 03:08:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:57.957Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"-.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689545263372
Jul 17 03:08:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:59.344Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_scaling_frequency_max_hertz\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", cpu=\"35\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"OLAP-Node-Exporter-pcap-dellr740-dt001\", endpoint_id=\"251\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545277557
Jul 17 03:08:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:59.757Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"msh-tsgx01\", asset_id=\"51\", container=\"ipmi-exporter\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"cadvisor\", failure_type=\"pgmajfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"hierarchy\"}" timestamp=1689545272624
Jul 17 03:08:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:59.774Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_info\", asset=\"msh-tsgx01\", asset_id=\"51\", container=\"helm\", container_id=\"containerd://b0e9ae73ed4e730d3ac50b00d95959c4ef8bf34e974bddf7d69c227126b48df8\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", image=\"docker.io/rancher/klipper-helm:v0.8.0-build20230510\", image_id=\"sha256:6f42df210d7faa0b6eb56f5f28bb061930c7834af1e7dd4b3586108ec6b1485b\", image_spec=\"rancher/klipper-helm:v0.8.0-build20230510\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-cm-policy-local-cache-9rh48\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"592a0d86-ef77-4a04-b590-aa94c9fe12b4\"}" timestamp=1689545267425
Jul 17 03:08:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:59.784Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-85717fa0db1a4398ce7ca9407c9d674452dc18da72ec382d99fafe98c495ef1b-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689545264879
Jul 17 03:08:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:59.798Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_edac_uncorrectable_errors_total\", asset=\"msh-tsgx01\", asset_id=\"51\", controller=\"0\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545269083
Jul 17 03:08:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:08:59.970Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_timex_pps_shift_seconds\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545269083
Jul 17 03:09:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:00.096Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_errors_received_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_10\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545269083
Jul 17 03:09:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:00.118Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"92\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545269083
Jul 17 03:09:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:00.124Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Tcp_Bypass_STM\", app_name=\"sapp4\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", vsysId=\"1\"}" timestamp=1689545273479
Jul 17 03:09:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:00.138Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545269083
Jul 17 03:09:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:00.141Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"nfs-server.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"oneshot\", type=\"service\"}" timestamp=1689545264879
Jul 17 03:09:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:00.171Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-255b4943de2ba3b9c612af6809614fa93a430e288d3ca20214387052f80fd2e4.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"scope\"}" timestamp=1689545264879
Jul 17 03:09:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:00.193Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"initrd-root-fs.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"target\"}" timestamp=1689545264879
Jul 17 03:09:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:00.223Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"promhttp_metric_handler_requests_in_flight\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545269083
Jul 17 03:09:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:00.224Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_packets_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"cadvisor\", interface=\"eno34np1\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545272512
Jul 17 03:09:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:00.243Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db28\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545265071
Jul 17 03:09:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:00.246Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"rpc-gssd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"forking\", type=\"service\"}" timestamp=1689545264879
Jul 17 03:09:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:00.310Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_scrape_collector_duration_seconds\", asset=\"msh-tsgx01\", asset_id=\"51\", collector=\"filesystem\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545269083
Jul 17 03:09:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:00.326Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-rfkill.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689545264879
Jul 17 03:09:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:00.380Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_avg_ttl_seconds\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db1\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545265071
Jul 17 03:09:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:00.383Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"multi-user.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"target\"}" timestamp=1689545264879
Jul 17 03:09:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:00.389Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_metadata_generation\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"traefik\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545267425
Jul 17 03:09:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:00.400Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Unknown_Pkt\", app_name=\"sapp4\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", vsysId=\"1\"}" timestamp=1689545268260
Jul 17 03:09:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:00.440Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_info\", asset=\"msh-tsgx01\", asset_id=\"51\", container=\"metrics\", container_id=\"containerd://286e9e2e78631e03427a533952967197d6ba74291280491099bf1d3700b1200c\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", image=\"docker.io/bitnami/redis-exporter:1.50.0\", image_id=\"sha256:ae434d812b4136c2a777882d508b25f384be532ea3e3686b3836bec55cf7b1e8\", image_spec=\"docker.io/bitnami/redis-exporter:1.50.0\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"cm-policy-local-cache-redis-master-69967b5879-kmxnw\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"afda43ab-d2e7-4daa-bc44-9edf3c57be96\"}" timestamp=1689545267425
Jul 17 03:09:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:00.483Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_memory_MemAvailable_bytes\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545269083
Jul 17 03:09:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:00.499Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_writes_total\", asset=\"msh-tsgx01\", asset_id=\"51\", container=\"firewall\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689545260416
Jul 17 03:09:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:00.568Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filefd_maximum\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545269083
Jul 17 03:09:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:00.584Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens3f0np0\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545269083
Jul 17 03:09:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:00.713Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"81\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545269083
Jul 17 03:09:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:01.063Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_type\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.configmap-reloader.v1\", service=\"prometheus-kube-state-metrics\", type=\"helm.sh/release.v1\"}" timestamp=1689545267425
Jul 17 03:09:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:01.179Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_size_bytes\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"shm\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/90bfaf448a4c3c7470b7af81c62e40408db991a36fbd23404409877f41f3d5f6/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545269083
Jul 17 03:09:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:02.501Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_fully_labeled_replicas\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"prometheus-kube-state-metrics-6b768c7bf8\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545271412
Jul 17 03:09:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:02.528Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"snmptrapd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689545268447
Jul 17 03:09:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:02.543Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"msh-tsgx02\", asset_id=\"63\", cpu=\"1\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545278318
Jul 17 03:09:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:02.564Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"slices.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"target\"}" timestamp=1689545268447
Jul 17 03:09:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:02.631Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:c0-0000:c0:01.1-0000:c1:00.1-infiniband-mlx5_9.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689545268447
Jul 17 03:09:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:02.701Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"slices.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"target\"}" timestamp=1689545268447
Jul 17 03:09:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:02.723Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_replicas\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-1-firewall-6b6bf4679c\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545271412
Jul 17 03:09:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:03.283Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_bytes_total\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", interface=\"eno1\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545266907
Jul 17 03:09:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:03.304Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx02\", asset_id=\"63\", container=\"firewall\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Write\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689545265931
Jul 17 03:09:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:03.355Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"prometheus_http_response_size_bytes_bucket\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", handler=\"/api/v1/targets\", instance=\"10.10.20.159:10090\", le=\"10000\", type=\"global\"}" timestamp=1689545282989
Jul 17 03:09:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:03.375Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_ips\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", ip=\"10.42.0.151\", ip_family=\"4\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"mrzcpd-exporter-5cf7f967cc-9zrnz\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"0b1be0e3-e71c-4f3b-83a7-1d29910105b8\"}" timestamp=1689545271412
Jul 17 03:09:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:03.522Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"126\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545278318
Jul 17 03:09:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:03.603Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx02\", asset_id=\"63\", cpu=\"22\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545278318
Jul 17 03:09:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:03.645Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_memory_Dirty_bytes\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-dt001\", endpoint_id=\"225\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545282988
Jul 17 03:09:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:03.937Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_NameNode_NumberOfMissingBlocks\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-Hadoop-msh-dellr740-dt001-NameNode\", endpoint_id=\"327\", module=\"DC-Hadoop\", module_id=\"53\", name=\"NameNodeInfo\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545283183
Jul 17 03:09:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:03.987Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_down_changes_total\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens4f1np1\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545278318
Jul 17 03:09:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:04.059Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"msh-tsgx02\", asset_id=\"63\", cpu=\"18\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545278318
Jul 17 03:09:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:04.072Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_commands_rejected_calls_total\", asset=\"msh-tsgx02\", asset_id=\"63\", cmd=\"multi\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545269269
Jul 17 03:09:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:04.111Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_start_time_seconds\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545269269
Jul 17 03:09:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:04.188Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_frequency_min_hertz\", asset=\"pcap-dellr740-dt002\", asset_id=\"33\", cpu=\"32\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"OLAP-Node-Exporter-pcap-dellr740-dt002\", endpoint_id=\"253\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545282497
Jul 17 03:09:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:04.243Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"auth-rpcgss-module.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689545268447
Jul 17 03:09:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:04.603Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"prometheus_target_scrape_pool_targets\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", instance=\"10.10.20.159:10090\", scrape_job=\"385\", type=\"global\"}" timestamp=1689545282989
Jul 17 03:09:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:06.815Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_annotations\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"local-path-provisioner-957fdf8bc\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545281227
Jul 17 03:09:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:06.830Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_drop_total\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth76f0c56a\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545283910
Jul 17 03:09:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:06.897Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db12\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545278290
Jul 17 03:09:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:06.945Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"gpm.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"service\"}" timestamp=1689545273582
Jul 17 03:09:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:07.120Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_protocol_type\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth5fb026f4\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545283910
Jul 17 03:09:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:07.167Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cpu=\"48\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545283910
Jul 17 03:09:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:07.572Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Udp_Offload_STM\", app_name=\"sapp4\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", vsysId=\"1\"}" timestamp=1689545277817
Jul 17 03:09:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:07.810Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"tmpfs\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/var/lib/kubelet/pods/083e9092-7400-4cad-afc0-cffbf05464a4/volumes/kubernetes.io~projected/kube-api-access-zh252\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545283910
Jul 17 03:09:09 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:09.978Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"clickhouse_network_receive_drop_em4\", asset=\"msh-dellr740-olap002\", asset_id=\"13\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Clickhouse-msh-dellr740-olap002\", endpoint_id=\"73\", module=\"NC-Clickhouse\", module_id=\"29\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545287758
Jul 17 03:09:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:11.022Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_compactionsQueuedCount\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"default\", project=\"OLAP\", project_id=\"3\", region=\"c74587d641410ee89476589ede4d7a58\", table=\"index_time_default\", type=\"index\"}" timestamp=1689545288804
Jul 17 03:09:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:11.769Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-dellr740-dt002\", asset_id=\"39\", cpu=\"23\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"NC-Gohangout-twa-dellr740-dt002\", endpoint_id=\"179\", mode=\"user\", module=\"DC-Gohangout\", module_id=\"35\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545291525
Jul 17 03:09:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:12.199Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_minStoreFileAge\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"00dc5a251addf7b4aaed51561226ada8\", table=\"proxy_hos_bucket\", type=\"data\"}" timestamp=1689545290149
Jul 17 03:09:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:12.200Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_compactionsCompletedCount\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"babc1946f31faf5b4151981143c005cf\", table=\"default\", type=\"data\"}" timestamp=1689545288804
Jul 17 03:09:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:12.229Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_numBytesCompactedCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"assessment_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"35933830b1d0022d0e7e07490958e85a\", table=\"index_partfile_assessment_hos_bucket\", type=\"index\"}" timestamp=1689545290149
Jul 17 03:09:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:12.644Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_tsg_table_report_result_metric_scanTime_25th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545290149
Jul 17 03:09:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:12.721Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_bytes_total\", asset=\"twa-dellr740-dt002\", asset_id=\"39\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth54187e7\", endpoint=\"NC-Gohangout-twa-dellr740-dt002\", endpoint_id=\"179\", module=\"DC-Gohangout\", module_id=\"35\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545291525
Jul 17 03:09:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:12.723Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_2_metric_appendTime_95th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545290149
Jul 17 03:09:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:12.828Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_1_metric_appendTime_num_ops\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545290149
Jul 17 03:09:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:12.883Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_2_metric_incrementTime_95th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545290149
Jul 17 03:09:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:12.971Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_blockCacheDeleteFamilyBloomHitCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Server\"}" timestamp=1689545290149
Jul 17 03:09:13 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:13.129Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_2_metric_deleteTime_num_ops\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545290149
Jul 17 03:09:13 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:13.273Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_compactionsQueuedCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"firewall_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"7825e373ea56537e53b94051524e36b0\", table=\"index_filename_firewall_hos_bucket\", type=\"index\"}" timestamp=1689545290149
Jul 17 03:09:13 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:13.287Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_readRequestCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"assessment_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"92a513a478a0fcdaa762082cf252dacc\", table=\"index_time_assessment_hos_bucket\", type=\"index\"}" timestamp=1689545290149
Jul 17 03:09:13 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:13.288Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_2_metric_storeFileSize\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545290149
Jul 17 03:09:13 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:13.402Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_blockCacheSize\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Memory\"}" timestamp=1689545290149
Jul 17 03:09:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:15.220Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_packets_transmitted_total\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_4\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545281230
Jul 17 03:09:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:15.672Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-pod18136b1d_dee5_4f2e_a352_07c6fe3b2565.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"slice\"}" timestamp=1689545287046
Jul 17 03:09:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:15.684Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-5d8ba54f071fbeea504df47b857740cc06184da019235853e66b2dc0d22ef817.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"scope\"}" timestamp=1689545287046
Jul 17 03:09:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:15.706Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2did-scsi\\\\x2d362cea7f0848271002c25f03b9c8b41c5\\\\x2dpart1.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689545287046
Jul 17 03:09:17 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:17.286Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:00-0000:00:01.1-0000:01:00.1-infiniband-mlx5_1.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689545287046
Jul 17 03:09:17 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:17.355Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_changes_total\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth5c296dad\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545281230
Jul 17 03:09:17 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:17.723Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_mapped_file\", asset=\"twa-tsgx07\", asset_id=\"103\", container=\"prometheus-server\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545271130
Jul 17 03:09:17 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:17.778Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"twa-tsgx07\", asset_id=\"103\", container=\"firewall\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", state=\"iowaiting\"}" timestamp=1689545280660
Jul 17 03:09:17 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:17.900Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_user_seconds_total\", asset=\"twa-tsgx07\", asset_id=\"103\", container=\"telegraf\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689545279604
Jul 17 03:09:17 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:17.981Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"124\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545281230
Jul 17 03:09:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:19.834Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"92\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545297612
Jul 17 03:09:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:19.972Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_fully_labeled_replicas\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"coredns-7bb96695bb\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545293949
Jul 17 03:09:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:20.051Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_textfile_scrape_error\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545297612
Jul 17 03:09:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:20.058Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"54\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545297612
Jul 17 03:09:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:20.081Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dracut-shutdown.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689545284356
Jul 17 03:09:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:20.212Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_address_assign_type\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth48720f3f\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545297612
Jul 17 03:09:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:20.335Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"pcap-tsgx10\", asset_id=\"89\", condition=\"unknown\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", pod=\"tsg-traffic-engine-numa-0-firewall-6d4969c56f-54wcn\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"617272f1-e26a-4d41-b763-554ea7e04473\"}" timestamp=1689545293828
Jul 17 03:09:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:20.336Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"99\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545288263
Jul 17 03:09:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:20.358Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"104\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545288263
Jul 17 03:09:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:20.385Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_constraint_errors_transmitted_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"mlx5_6\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545288263
Jul 17 03:09:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:20.392Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_created\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-prometheus-fmhhw\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"4f84a780-549d-4f40-bdec-aee8906d8427\"}" timestamp=1689545293828
Jul 17 03:09:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:20.414Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_annotations\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.traefik-crd.v1\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545293828
Jul 17 03:09:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:20.429Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"clevis-luks-askpass.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"service\"}" timestamp=1689545290657
Jul 17 03:09:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:20.452Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_replicas\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-1-firewall-57fdfbfc9f\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545293828
Jul 17 03:09:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:20.488Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-0cbba4d97a73cf1cf108a44b70aecad491f4b19a4049279ba4e8f9238ce32d97-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689545290657
Jul 17 03:09:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:20.728Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"11\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545288263
Jul 17 03:09:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:20.778Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-server-6ff784d758-wkch9\", project=\"TSG-OS\", project_id=\"7\", reason=\"NodeAffinity\", service=\"prometheus-kube-state-metrics\", uid=\"c8175cdd-27fb-4976-a041-ad5b277f4e45\"}" timestamp=1689545293828
Jul 17 03:09:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:20.801Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"47\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545288263
Jul 17 03:09:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:20.880Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_labels\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"metrics-server\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545293828
Jul 17 03:09:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:20.906Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_packets_dropped_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"cadvisor\", interface=\"ens3f0np0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545293933
Jul 17 03:09:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:20.916Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"87\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545288263
Jul 17 03:09:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:21.081Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-4596f966725b70f54ddf02e19aacb92d52d5b8902858bd05bd7fd2c3ef16cbbe.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"scope\"}" timestamp=1689545290657
Jul 17 03:09:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:21.119Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"svclb-traefik-2b43699a-xnrnl\", project=\"TSG-OS\", project_id=\"7\", reason=\"UnexpectedAdmissionError\", service=\"prometheus-kube-state-metrics\", uid=\"d430cb8f-d3b8-4233-a528-300590ccd3d3\"}" timestamp=1689545293949
Jul 17 03:09:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:21.126Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"6\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545297612
Jul 17 03:09:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:21.143Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"paths.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"target\"}" timestamp=1689545284356
Jul 17 03:09:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:21.157Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx06\", asset_id=\"101\", container=\"dashboard-metrics-scraper\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Async\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545284257
Jul 17 03:09:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:21.168Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_ready\", asset=\"twa-tsgx06\", asset_id=\"101\", container=\"helm\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-mrzcpd-exporter-2hpgf\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"e068cb4e-a336-4c6c-ab42-0a667f0f7cda\"}" timestamp=1689545293949
Jul 17 03:09:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:21.185Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"64\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545297612
Jul 17 03:09:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:21.273Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx07\", asset_id=\"53\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx07\", endpoint_id=\"421\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"nfs-utils.service\", nz_agent_id=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"service\"}" timestamp=1689545291532
Jul 17 03:09:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:21.293Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_protocol_type\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth9421f8b7\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545297612
Jul 17 03:09:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:21.354Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_fifo_total\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth0ae4eb9d\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545297612
Jul 17 03:09:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:21.369Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2did-wwn\\\\x2d0x62cea7f0848275002c26f81cd27e8046\\\\x2dpart5.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689545284356
Jul 17 03:09:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:21.659Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_owner\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", owner_is_controller=\"true\", owner_kind=\"ReplicaSet\", owner_name=\"tsg-traffic-engine-numa-1-firewall-5d7bfbbbfb\", pod=\"tsg-traffic-engine-numa-1-firewall-5d7bfbbbfb-4wgxr\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"ab53e5a7-f22e-498d-b5af-535ae7359350\"}" timestamp=1689545293828
Jul 17 03:09:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:21.794Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_processes\", asset=\"pcap-tsgx10\", asset_id=\"89\", container=\"telegraf\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689545277990
Jul 17 03:09:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:21.808Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_packets_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"cadvisor\", interface=\"ens4f0np0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545293933
Jul 17 03:09:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:21.931Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dnf-makecache.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689545290657
Jul 17 03:09:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:22.005Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_spec_parallelism\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", job_name=\"helm-install-tsg-traffic-engine-numa-0\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545293828
Jul 17 03:09:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:22.045Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"53\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545288263
Jul 17 03:09:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:22.074Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_labels\", asset=\"pcap-tsgx10\", asset_id=\"89\", configmap=\"local-path-config\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545293828
Jul 17 03:09:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:22.115Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"traefik\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545293828
Jul 17 03:09:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:22.118Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Succeeded\", pod=\"traefik-64f55bb67d-v57x6\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"eb9c8310-844c-4b72-9e0c-d86e20973848\"}" timestamp=1689545293828
Jul 17 03:09:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:22.183Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"104\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545288263
Jul 17 03:09:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:22.444Z caller=dedupe.go:112 component=remote level=error remote_name=52db6f url=http://10.10.10.159:10090/nz-agent/prometheus/proxy/api/v1/write msg="non-recoverable error" count=1000 exemplarCount=0 err="server returned HTTP status 400 : out of order sample"
Jul 17 03:09:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:23.655Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"msh-tsgx09\", asset_id=\"55\", condition=\"true\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-traefik-crd-tjh7p\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"2a496308-3fd5-494e-83f9-537bb34e5898\"}" timestamp=1689545300068
Jul 17 03:09:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:23.681Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"msh-tsgx09\", asset_id=\"55\", condition=\"Available\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"ipmi-exporter\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"unknown\"}" timestamp=1689545300068
Jul 17 03:09:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:23.720Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Udp_Bit\", app_name=\"sapp4\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", vsysId=\"1\"}" timestamp=1689545298016
Jul 17 03:09:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:23.780Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx09\", asset_id=\"55\", cpu=\"90\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545294790
Jul 17 03:09:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:23.995Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_swap\", asset=\"msh-tsgx09\", asset_id=\"55\", container=\"prometheus-pushgateway\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545280726
Jul 17 03:09:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:24.037Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx09\", asset_id=\"55\", cpu=\"16\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545294790
Jul 17 03:09:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:24.070Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_last_seen\", asset=\"msh-tsgx09\", asset_id=\"55\", container=\"lb-tcp-443\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545298606
Jul 17 03:09:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:24.099Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_cache\", asset=\"msh-tsgx09\", asset_id=\"55\", container=\"firewall\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689545285204
Jul 17 03:09:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:24.112Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_load_average_10s\", asset=\"msh-tsgx09\", asset_id=\"55\", container=\"mrapm-device\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545285377
Jul 17 03:09:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:24.124Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_cpu_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-podab6b1591_cfaa_4964_8fcc_a861d0862792.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"slice\"}" timestamp=1689545291102
Jul 17 03:09:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:24.134Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failcnt\", asset=\"msh-tsgx09\", asset_id=\"55\", container=\"traefik\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545290990
Jul 17 03:09:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:24.140Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_errors_total\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", interface=\"ens8f0np0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545295914
Jul 17 03:09:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:24.150Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_status_succeeded\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", job_name=\"helm-install-traefik-crd\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545300068
Jul 17 03:09:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:24.162Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2dpartlabel-TSG\\\\x2dOS\\\\x2dUPDATE.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689545298416
Jul 17 03:09:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:24.179Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx09\", asset_id=\"55\", cpu=\"127\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545294790
Jul 17 03:09:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:24.180Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_complete\", asset=\"msh-tsgx09\", asset_id=\"55\", condition=\"false\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", job_name=\"helm-delete-tsg-traffic-engine-numa-1\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545300068
Jul 17 03:09:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:24.228Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx09\", asset_id=\"55\", cpu=\"16\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545294790
Jul 17 03:09:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:24.243Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_speed_bytes\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"eno34np1\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545294790
Jul 17 03:09:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:24.249Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_type\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.cm-policy-local-cache.v1\", service=\"prometheus-kube-state-metrics\", type=\"helm.sh/release.v1\"}" timestamp=1689545291122
Jul 17 03:09:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:24.268Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"17\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545300160
Jul 17 03:09:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:24.298Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"tuned.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"dbus\", type=\"service\"}" timestamp=1689545291102
Jul 17 03:09:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:24.323Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"97\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545300160
Jul 17 03:09:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:24.431Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_ready\", asset=\"msh-tsgx09\", asset_id=\"55\", container=\"helm\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-configmap-reloader-84s59\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"47ba69e1-fd09-4e39-941a-89000685484e\"}" timestamp=1689545300068
Jul 17 03:09:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:24.440Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx09\", asset_id=\"55\", cpu=\"88\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545294790
Jul 17 03:09:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:24.482Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_last_terminated_reason\", asset=\"msh-tsgx09\", asset_id=\"55\", container=\"dashboard-metrics-scraper\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", pod=\"dashboard-metrics-scraper-5b94897dc9-t6qds\", project=\"TSG-OS\", project_id=\"7\", reason=\"Unknown\", service=\"prometheus-kube-state-metrics\", uid=\"9ab87c24-ae10-4bca-bc1b-c09bbe929130\"}" timestamp=1689545300068
Jul 17 03:09:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:24.498Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth4a8b9971\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545294790
Jul 17 03:09:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:24.552Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"24\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545300160
Jul 17 03:09:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:24.577Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"local-path-provisioner\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"uninterruptible\"}" timestamp=1689545301624
Jul 17 03:09:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:24.592Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_state_started\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"dashboard-metrics-scraper\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", pod=\"dashboard-metrics-scraper-5b94897dc9-hgw87\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"b65f87c9-c665-4fc6-9a65-8ece7ccf5f5a\"}" timestamp=1689545291122
Jul 17 03:09:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:24.608Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"83\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545300160
Jul 17 03:09:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:24.620Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_annotations\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", job_name=\"helm-install-ipmi-exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545291122
Jul 17 03:09:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:24.666Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_init_container_status_restarts_total\", asset=\"msh-tsgx09\", asset_id=\"55\", container=\"firewall-init\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", pod=\"tsg-traffic-engine-numa-1-firewall-5f74b447d7-bwdnr\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"f2551d6b-0939-434d-ba32-2e369c56471c\"}" timestamp=1689545300068
Jul 17 03:09:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:24.672Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx09\", asset_id=\"55\", cpu=\"109\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545294790
Jul 17 03:09:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:24.682Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_labels\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"chart-values-prometheus\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545300068
Jul 17 03:09:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:24.695Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"81\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545300160
Jul 17 03:09:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:24.697Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_speed_bytes\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"eno33np0\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545294790
Jul 17 03:09:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:24.713Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:c0-0000:c0:01.1-0000:c1:00.1-infiniband_mad-umad9.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689545291102
Jul 17 03:09:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:24.714Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"vdo.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"oneshot\", type=\"service\"}" timestamp=1689545298416
Jul 17 03:09:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:24.725Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-veth168605c4.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689545291102
Jul 17 03:09:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:24.978Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_aof_rewrite_in_progress\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545301795
Jul 17 03:09:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:25.016Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db239\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545301795
Jul 17 03:09:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:25.049Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_resident_memory_bytes\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"exporter-systemd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689545291102
Jul 17 03:09:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:25.231Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_tasks_max\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"rshim.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689545291102
Jul 17 03:09:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:25.236Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_bytes_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", interface=\"eno35np2\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545302966
Jul 17 03:09:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:26.106Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_size_bytes\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"shm\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/eda31d8d78c9a278bed24273dc8c3f52f6fa296de36fcab3f456c853734b5e12/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545295810
Jul 17 03:09:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:26.108Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"5\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545295810
Jul 17 03:09:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:26.161Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"twa-tsgx09\", asset_id=\"107\", condition=\"false\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-pushgateway-bd476bcf9-zqlv8\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"880aa12b-51b0-4413-88cb-d06b2fc6d0c3\"}" timestamp=1689545298744
Jul 17 03:09:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:26.188Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled_time\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"traefik-64f55bb67d-s7cwm\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"80e5751c-f8e9-49bd-ae9e-664fe9b134da\"}" timestamp=1689545298744
Jul 17 03:09:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:26.223Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"20\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545295810
Jul 17 03:09:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:26.251Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_memory_Active_file_bytes\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545295810
Jul 17 03:09:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:26.426Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-netns-cni\\\\x2d38126a27\\\\x2d5429\\\\x2db601\\\\x2d7677\\\\x2d2ff89ef17ecb.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689545293907
Jul 17 03:09:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:26.589Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_inodes_total\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda4\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545303522
Jul 17 03:09:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:26.607Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_processes\", asset=\"twa-tsgx09\", asset_id=\"107\", container=\"metrics-server\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545288738
Jul 17 03:09:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:26.724Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"118\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545295810
Jul 17 03:09:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:26.747Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-e79a435c\\\\x2d7f48\\\\x2d4427\\\\x2d9500\\\\x2d7dd3dac82d83-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2dxzr8q.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689545293907
Jul 17 03:09:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:26.761Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"iscsiuio.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"service\"}" timestamp=1689545293907
Jul 17 03:09:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:26.816Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_ulimits_soft\", asset=\"twa-tsgx09\", asset_id=\"107\", container=\"prometheus-pushgateway\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", ulimit=\"max_open_files\"}" timestamp=1689545298541
Jul 17 03:09:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:26.856Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_metadata_generation\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"prometheus-server\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545298744
Jul 17 03:09:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:26.892Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sysstat-summary.timer\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"timer\"}" timestamp=1689545293907
Jul 17 03:09:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:27.008Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"25\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545295810
Jul 17 03:09:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:27.143Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_readonly\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda2\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", fstype=\"ext4\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/mnt/onie-boot\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545295810
Jul 17 03:09:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:27.155Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_constraint_errors_transmitted_total\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_2\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545295810
Jul 17 03:09:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:27.388Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"84\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545302882
Jul 17 03:09:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:27.404Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"75\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545302882
Jul 17 03:09:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:27.450Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_oom_events_total\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545305155
Jul 17 03:09:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:27.463Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_cpu_shares\", asset=\"twa-tsgx09\", asset_id=\"107\", container=\"redis\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545305490
Jul 17 03:09:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:27.471Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_metadata_resource_version\", asset=\"twa-tsgx09\", asset_id=\"107\", configmap=\"kube-root-ca.crt\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545298744
Jul 17 03:09:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:27.480Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_start_time_seconds\", asset=\"twa-tsgx09\", asset_id=\"107\", container=\"kubernetes-dashboard\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545305490
Jul 17 03:09:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:27.485Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_ready_replicas\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-1-firewall-58d457d665\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545298744
Jul 17 03:09:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:27.511Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_owner\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", owner_is_controller=\"true\", owner_kind=\"Job\", owner_name=\"helm-install-cm-policy-local-cache\", pod=\"helm-install-cm-policy-local-cache-wzw8p\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"57683893-10c5-486b-89a6-dbc1429ed965\"}" timestamp=1689545298744
Jul 17 03:09:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:27.531Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_type\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"chart-values-cm-policy-local-cache\", service=\"prometheus-kube-state-metrics\", type=\"Opaque\"}" timestamp=1689545298744
Jul 17 03:09:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:27.568Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_packets_dropped_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", interface=\"eth0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689545298428
Jul 17 03:09:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:27.581Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-0b0292d1\\\\x2d7c6a\\\\x2d4fad\\\\x2da1f9\\\\x2df14e3c7cb4f8-volume\\\\x2dsubpaths-config\\\\x2dvolume-telegraf\\\\x2dsecurity-2.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689545294114
Jul 17 03:09:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:27.619Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_mtu_bytes\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"br_dign_s\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545302882
Jul 17 03:09:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:27.655Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_created\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"chart-values-systemd-exporter\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545295682
Jul 17 03:09:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:27.702Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_link_error_recovery_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_9\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545302882
Jul 17 03:09:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:27.718Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_memory_NFS_Unstable_bytes\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545302882
Jul 17 03:09:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:27.730Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"local-path-provisioner-957fdf8bc-hswhg\", project=\"TSG-OS\", project_id=\"7\", reason=\"NodeLost\", service=\"prometheus-kube-state-metrics\", uid=\"99c87350-195b-4ca1-93bd-9a804bf4d300\"}" timestamp=1689545295682
Jul 17 03:09:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:27.744Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:20-0000:20:03.1-0000:21:00.0-net-ens2f0np0.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689545294114
Jul 17 03:09:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:27.822Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"104\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545302882
Jul 17 03:09:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:27.843Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_bytes_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth875ea1eb\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545302882
Jul 17 03:09:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:27.873Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_packets_dropped_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", interface=\"eno33np0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545303455
Jul 17 03:09:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:27.906Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"14\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545302882
Jul 17 03:09:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:27.910Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_drop_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens4f3np3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545302882
Jul 17 03:09:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:27.930Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_colls_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens6f0np0\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545302882
Jul 17 03:09:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:27.948Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"redis\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"uninterruptible\"}" timestamp=1689545303530
Jul 17 03:09:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:27.964Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_annotations\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"kube-dns\", uid=\"38dc23a0-302e-4ea5-bde1-d09f51d29b58\"}" timestamp=1689545295682
Jul 17 03:09:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:28.236Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_rss\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"exporter\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545291081
Jul 17 03:09:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:28.297Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_errors_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", interface=\"ens8f1np1\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545303455
Jul 17 03:09:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:28.559Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_working_set_bytes\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"dign-client\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545293508
Jul 17 03:09:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:28.579Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_compressed_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"eno35np2\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545302882
Jul 17 03:09:30 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:30.711Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kafka_network_RequestMetrics_Errors_total\", asset=\"twa-dellr740-dt002\", asset_id=\"39\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-Kafka-twa-dellr740-dt002\", endpoint_id=\"107\", error=\"NONE\", module=\"DC-Kafka\", module_id=\"5\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", request=\"ApiVersions\"}" timestamp=1689545309472
Jul 17 03:09:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:31.016Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"ext4\", name=\"var-lib-kubelet-pods-99e7b7f0\\\\x2d307f\\\\x2d4274\\\\x2da670\\\\x2dfaf429f178f7-volume\\\\x2dsubpaths-config\\\\x2dvolume-telegraf-2.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689545301015
Jul 17 03:09:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:31.032Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"user.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"slice\"}" timestamp=1689545301015
Jul 17 03:09:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:31.107Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"process_max_fds\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545310141
Jul 17 03:09:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:31.120Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_compressed_total\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth8163644e\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545300939
Jul 17 03:09:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:31.143Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db119\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545310141
Jul 17 03:09:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:31.204Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"iscsi-shutdown.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"service\"}" timestamp=1689545301015
Jul 17 03:09:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:31.259Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_max_usage_bytes\", asset=\"msh-tsgx05\", asset_id=\"67\", container=\"telegraf-security\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689545291205
Jul 17 03:09:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:31.296Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"79\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545300939
Jul 17 03:09:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:31.364Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_memory_DirectMap1G_bytes\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545300939
Jul 17 03:09:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:31.592Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"emergency.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"service\"}" timestamp=1689545301015
Jul 17 03:09:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:31.599Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"16\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545299093
Jul 17 03:09:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:31.618Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Udp_Link_Del\", app_name=\"sapp4\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", vsysId=\"1\"}" timestamp=1689545309611
Jul 17 03:09:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:31.706Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"115\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545300939
Jul 17 03:09:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:31.737Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_local_link_integrity_errors_total\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_6\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545300939
Jul 17 03:09:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:31.787Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"81\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545299093
Jul 17 03:09:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:31.788Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", phase=\"Unknown\", pod=\"prometheus-alertmanager-6df7f457b6-7vmmh\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"a69122b9-4d3a-46e4-a296-9ed5e1d8b954\"}" timestamp=1689545304068
Jul 17 03:09:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:31.814Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_gc_duration_seconds\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", quantile=\"0.75\", service=\"prometheus-node-exporter\"}" timestamp=1689545299093
Jul 17 03:09:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:31.822Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_created\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"shaping-announce-port-tsg-traffic-engine-numa-0\", uid=\"f4d152fc-bbdb-4dc3-a62f-acd7a4f0591a\"}" timestamp=1689545304068
Jul 17 03:09:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:31.859Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"53\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545300939
Jul 17 03:09:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:31.860Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"twa-tsgx03\", asset_id=\"95\", condition=\"unknown\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-node-exporter-dd5xk\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"18031228-abd9-478c-bf5c-0acdbff9f2e6\"}" timestamp=1689545310436
Jul 17 03:09:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:31.902Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Udp_Close_byK\", app_name=\"sapp4\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", vsysId=\"1\"}" timestamp=1689545300031
Jul 17 03:09:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:31.995Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_data_received_bytes_total\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_6\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545299093
Jul 17 03:09:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:32.038Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-infiniband-umad6.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689545302644
Jul 17 03:09:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:32.067Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"52\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545299093
Jul 17 03:09:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:32.078Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_bytes_total\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens7f1np1\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545300791
Jul 17 03:09:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:32.217Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"91\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545300791
Jul 17 03:09:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:32.272Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"120\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545300791
Jul 17 03:09:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:32.300Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-infiniband-umad3.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689545302644
Jul 17 03:09:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:32.388Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_files_free\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"shm\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/e7ef0f6f76aa9c08b93f4f2e538436f3e9ab662e80c1533b8e2a80f554e3a0da/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545300791
Jul 17 03:09:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:32.412Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"35\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545299093
Jul 17 03:09:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:32.445Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"shm\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/a632288c817034ef9afe274ba19e12b639c521505775aedb4b4d36a71848dc5b/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545299093
Jul 17 03:09:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:32.453Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"80\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545300791
Jul 17 03:09:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:32.457Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"116\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545299093
Jul 17 03:09:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:32.480Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_up\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"virtio_dign_s\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545299093
Jul 17 03:09:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:32.490Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"iscsid.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689545306907
Jul 17 03:09:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:32.490Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"NetworkManager-wait-online.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689545302644
Jul 17 03:09:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:32.527Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-4c6ffbcaa8b131a32308a9f21af2e6702d7728dfa06f9b780d1d23227c3735db-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689545306907
Jul 17 03:09:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:32.890Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-2096ff4a\\\\x2d92d2\\\\x2d47a7\\\\x2d9b70\\\\x2d293a6568cee4-volume\\\\x2dsubpaths-config\\\\x2dpath-dign\\\\x2dclient-3.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689545302644
Jul 17 03:09:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:32.892Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545300791
Jul 17 03:09:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:32.911Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"122\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545300791
Jul 17 03:09:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:32.941Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Dup_Tcp_Pkt\", app_name=\"sapp4\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", vsysId=\"1\"}" timestamp=1689545307477
Jul 17 03:09:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:32.958Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sysstat.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689545302644
Jul 17 03:09:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:33.012Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-virtual-net-vetheef6d56c.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689545302644
Jul 17 03:09:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:33.072Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_annotations\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-1-firewall-85ff459dfc\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545299894
Jul 17 03:09:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:33.092Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_packets_total\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth50f7b1ca\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545300605
Jul 17 03:09:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:33.134Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"twa-tsgx05\", asset_id=\"99\", container=\"kube-state-metrics\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"uninterruptible\"}" timestamp=1689545291870
Jul 17 03:09:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:33.608Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_limit_bytes\", asset=\"twa-tsgx05\", asset_id=\"99\", container=\"mrapm-device\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545298535
Jul 17 03:09:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:33.646Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"28\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545300605
Jul 17 03:09:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:33.704Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_mapped_file\", asset=\"twa-tsgx05\", asset_id=\"99\", container=\"local-path-provisioner\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545292097
Jul 17 03:09:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:33.766Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_frame_total\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth2f4f6888\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545300605
Jul 17 03:09:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:33.826Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_oom_events_total\", asset=\"twa-tsgx05\", asset_id=\"99\", container=\"monit-stream\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545295266
Jul 17 03:09:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:33.840Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_spec_completions\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", job_name=\"helm-install-configmap-reloader\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545299894
Jul 17 03:09:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:33.935Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"basic.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"target\"}" timestamp=1689545307691
Jul 17 03:09:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:40.542Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_compressed_total\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens8f1np1\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545311647
Jul 17 03:09:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:40.570Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_drop_total\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"lo\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545311647
Jul 17 03:09:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:40.574Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_fully_labeled_replicas\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"mrzcpd-exporter-5cf7f967cc\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545320034
Jul 17 03:09:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:40.575Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"46\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545311647
Jul 17 03:09:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:40.609Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"twa-tsgx04\", asset_id=\"97\", container=\"monit-stream\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"cadvisor\", failure_type=\"pgfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"container\"}" timestamp=1689545296527
Jul 17 03:09:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:40.629Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"twa-tsgx04\", asset_id=\"97\", condition=\"Available\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"cm-policy-local-cache-redis-master\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"false\"}" timestamp=1689545320034
Jul 17 03:09:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:40.632Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"85\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545311647
Jul 17 03:09:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:40.642Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_data_received_bytes_total\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_11\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545311647
Jul 17 03:09:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:40.833Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_compactedCellsSize\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Server\"}" timestamp=1689545319756
Jul 17 03:09:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:40.863Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_putCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"troubleshooting_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"ecdd5c825bfbcbc187cc17fcd647a69d\", table=\"index_filename_troubleshooting_hos_bucket\", type=\"index\"}" timestamp=1689545319756
Jul 17 03:09:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:40.882Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_compactionsCompletedCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"proxy_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"d50358d51b185086f76e71418eb6ac7d\", table=\"index_time_proxy_hos_bucket\", type=\"index\"}" timestamp=1689545319756
Jul 17 03:09:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:40.912Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_CATALOG_metric_getTime_98th_percentile\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545319756
Jul 17 03:09:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:40.972Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_mtu_bytes\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth3d4cfc49\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545311647
Jul 17 03:09:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:40.990Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-f613230c64a97d59841f81270226287f35a940049220f15759980486e8699188-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689545315764
Jul 17 03:09:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:41.061Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"twa-tsgx04\", asset_id=\"97\", condition=\"Available\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"prometheus-server\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"false\"}" timestamp=1689545320034
Jul 17 03:09:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:41.085Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_boot_time_seconds\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545311647
Jul 17 03:09:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:41.109Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_local_link_integrity_errors_total\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_1\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545311647
Jul 17 03:09:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:41.126Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545311647
Jul 17 03:09:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:41.518Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_10_194_1_metric_splitSuccessCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545319756
Jul 17 03:09:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:42.217Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas_ready\", asset=\"msh-tsgx06\", asset_id=\"61\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"dashboard-metrics-scraper\", endpoint=\"TSG-OS-msh-tsgx06\", endpoint_id=\"429\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545316928
Jul 17 03:09:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:42.533Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"pcap-tsgx01\", asset_id=\"71\", condition=\"Available\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"prometheus-alertmanager\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"true\"}" timestamp=1689545315154
Jul 17 03:09:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:42.545Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_created\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"chart-values-prometheus\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545315154
Jul 17 03:09:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:42.558Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link_mode\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth841d6801\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545316242
Jul 17 03:09:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:42.562Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"tmpfs\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/data/var/lib/kubelet/pods/e10cda11-8be1-4b90-a6b7-1a1567a187aa/volumes/kubernetes.io~projected/kube-api-access-4zwcp\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545316242
Jul 17 03:09:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:42.572Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"slice\"}" timestamp=1689545320671
Jul 17 03:09:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:42.577Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", cpu=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545316242
Jul 17 03:09:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:42.616Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", cpu=\"16\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545316242
Jul 17 03:09:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:42.628Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", container=\"redis\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"stopped\"}" timestamp=1689545316770
Jul 17 03:09:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:42.647Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"pcap-tsgx01\", asset_id=\"71\", condition=\"false\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"local-path-provisioner-957fdf8bc-r2qbd\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"4ff67ed4-4465-4d8e-8bc9-8a9bac607a46\"}" timestamp=1689545315154
Jul 17 03:09:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:42.655Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_annotations\", asset=\"pcap-tsgx01\", asset_id=\"71\", configmap=\"chart-content-tsg-traffic-engine-numa-0\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545315154
Jul 17 03:09:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:42.656Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_FlushTime_95th_percentile\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Server\"}" timestamp=1689545319756
Jul 17 03:09:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:42.757Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_fully_labeled_replicas\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"kubernetes-dashboard-66dd5b456\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545315154
Jul 17 03:09:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:42.856Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"pcap-tsgx01\", asset_id=\"71\", condition=\"true\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"svclb-traefik-7eb8511e-q59w6\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"0a82591e-01f6-43ee-9af0-f8cc0036ad27\"}" timestamp=1689545315154
Jul 17 03:09:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:42.876Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_up_changes_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth585262fb\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545316242
Jul 17 03:09:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:42.971Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", cpu=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545316242
Jul 17 03:09:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:43.031Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"127\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545316242
Jul 17 03:09:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:43.084Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", cpu=\"44\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545316242
Jul 17 03:09:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:43.144Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"mysql_global_status_commands_total\", asset=\"msh-dellr740-olap002\", asset_id=\"13\", command=\"show_create_package_body\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Mariadb-msh-dellr740-olap002\", endpoint_id=\"69\", module=\"NC-Mariadb\", module_id=\"43\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545322786
Jul 17 03:09:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:43.159Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", cpu=\"112\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545316242
Jul 17 03:09:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:43.162Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_annotations\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-0-firewall-6d9d5c4b6c\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545315154
Jul 17 03:09:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:43.174Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", container=\"lb-tcp-80\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"uninterruptible\"}" timestamp=1689545320960
Jul 17 03:09:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:43.177Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-module-fuse.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689545320671
Jul 17 03:09:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:43.182Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", cpu=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545316242
Jul 17 03:09:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:43.197Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_size_bytes\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda2\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", fstype=\"ext4\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/mnt/onie-boot\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545316242
Jul 17 03:09:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:43.246Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"overlay\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-7fd39b22aeb9c7c3c756d0cfc15763f1c2e8d87c33e18a1b2c854c5ccd92f347-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689545320671
Jul 17 03:09:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:43.542Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"mrzcpd-exporter-5cf7f967cc-sgsdt\", project=\"TSG-OS\", project_id=\"7\", reason=\"NodeAffinity\", service=\"prometheus-kube-state-metrics\", uid=\"23a4edf9-c51a-4ccf-aa70-4b82ecf894c2\"}" timestamp=1689545315154
Jul 17 03:09:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:43.562Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_labels\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.ipmi-exporter.v1\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545315154
Jul 17 03:09:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:43.584Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_fifo_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth2ab79c7d\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545316242
Jul 17 03:09:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:43.604Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", cpu=\"63\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545316242
Jul 17 03:09:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:43.614Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", cpu=\"69\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545316242
Jul 17 03:09:48 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:48.594Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_readRequestCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", namespace=\"SYSTEM\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"5a8f51f1ceb29e2c5e26a5988d4dce97\", table=\"LOG\", type=\"data\"}" timestamp=1689545327438
Jul 17 03:09:48 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:48.635Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_getCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"session_record_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"3e1c7aa641482ceb8e28b5526f23c5e9\", table=\"index_filename_session_record_hos_bucket\", type=\"index\"}" timestamp=1689545327438
Jul 17 03:09:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:49.318Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_numFilesCompactedCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"rtp_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"9866f735d37d21d8f3c1547005efc196\", table=\"index_partfile_rtp_hos_bucket\", type=\"index\"}" timestamp=1689545327438
Jul 17 03:09:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:49.456Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_default_metric_compactionInputSize_median\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545327438
Jul 17 03:09:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:49.516Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_readRequestCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"session_record_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"5e4a8e5af9d1dd712c1dd2749d210276\", table=\"index_partfile_session_record_hos_bucket\", type=\"index\"}" timestamp=1689545327438
Jul 17 03:09:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:49.692Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth23566533\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545321797
Jul 17 03:09:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:49.779Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_last_seen\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"firewall\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689545319080
Jul 17 03:09:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:49.803Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_last_seen\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545319086
Jul 17 03:09:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:49.873Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_max_usage_bytes\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"mrapm-device\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545312114
Jul 17 03:09:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:49.892Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_restart_policy\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-ipmi-exporter-bbkqn\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", type=\"OnFailure\", uid=\"af48b541-cb04-486f-951a-b49f9ad44dd5\"}" timestamp=1689545319668
Jul 17 03:09:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:49.974Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_default_metric_majorCompactionOutputSize_95th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545327438
Jul 17 03:09:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:49.990Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_maxFlushQueueSize\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", namespace=\"SYSTEM\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"41a022efe1c8646a317cd32a27b21bf0\", table=\"LOG\", type=\"data\"}" timestamp=1689545327438
Jul 17 03:09:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:50.321Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_files\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"shm\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/01c70c00c73026bd95e093fd125fbfe7edd2cf8ddef27e6fc330f43aa95ea77b/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545321797
Jul 17 03:09:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:50.351Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2did-scsi\\\\x2dSDELL_PERC_H345_Adp_00f5c79fdded05272c008ed5adf0a7ce\\\\x2dpart2.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689545321057
Jul 17 03:09:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:50.351Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-sda2.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689545321057
Jul 17 03:09:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:50.363Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_state_started\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"exporter-merger\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"mrzcpd-exporter-5cf7f967cc-mp8pv\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"71f6269b-5154-46cd-b538-223c28b8f982\"}" timestamp=1689545319668
Jul 17 03:09:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:50.388Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", cpu=\"45\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-olap003\", endpoint_id=\"235\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"yarn_resourcemanager,zookeeper,nacos,kafka,spark,druid_data,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545329713
Jul 17 03:09:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:50.399Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", fstype=\"ext4\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/var/lib/kubelet/pods/8bb79bea-895f-4fbf-bba7-ff346d91b1a0/volume-subpaths/config-volume/telegraf-security/2\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545321797
Jul 17 03:09:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:50.417Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_writes_merged_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/57586ef2b12d3be3dde6eb77d147247c3bda45d0866adf6b2d51c2372d8963e5/shm\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545325224
Jul 17 03:09:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:50.437Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_persistentvolumeclaim_annotations\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", persistentvolumeclaim=\"prometheus-server\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545321568
Jul 17 03:09:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:50.466Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"twa-tsgx08\", asset_id=\"105\", container=\"prometheus-alertmanager\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", failure_type=\"pgfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"hierarchy\"}" timestamp=1689545318112
Jul 17 03:09:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:50.501Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_up_changes_total\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"eno34np1\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545321797
Jul 17 03:09:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:50.505Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Read\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545325281
Jul 17 03:09:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:50.545Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_annotations\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-server-6ff784d758-mrg7n\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"a43b9675-fdba-4957-ab2b-41d084fe86c7\"}" timestamp=1689545321568
Jul 17 03:09:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:50.624Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_metadata_generation\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"dashboard-metrics-scraper-5b94897dc9\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545321568
Jul 17 03:09:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:50.684Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_ready\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"local-path-provisioner\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"local-path-provisioner-957fdf8bc-5w5bc\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"28e75c4d-4cd3-41cb-8435-dd13365e6e7d\"}" timestamp=1689545319668
Jul 17 03:09:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:50.691Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"kube-multus-ds-p9mt7\", project=\"TSG-OS\", project_id=\"7\", reason=\"Evicted\", service=\"prometheus-kube-state-metrics\", uid=\"f930767b-550c-482d-9c10-4ec670104c56\"}" timestamp=1689545319668
Jul 17 03:09:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:50.783Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_info\", address=\"96:78:64:74:0f:83\", asset=\"twa-tsgx08\", asset_id=\"105\", broadcast=\"ff:ff:ff:ff:ff:ff\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"virtio_dign_c\", duplex=\"full\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operstate=\"unknown\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545326709
Jul 17 03:09:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:50.848Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_rss\", asset=\"twa-tsgx08\", asset_id=\"105\", container=\"local-path-provisioner\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545326345
Jul 17 03:09:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:50.883Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx10\", asset_id=\"109\", cpu=\"47\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545321797
Jul 17 03:09:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:51.856Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_io_current\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/var/lib/kubelet/pods/2cfe23ce-7e71-44dc-8e19-b936dd7c7f8c/volumes/kubernetes.io~projected/kube-api-access-msvmx\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545325224
Jul 17 03:09:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:52.634Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", cpu=\"107\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545329337
Jul 17 03:09:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:52.647Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"initrd-udevadm-cleanup-db.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689545325496
Jul 17 03:09:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:52.931Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_queue_length\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth561c9aba\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545329337
Jul 17 03:09:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:52.948Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", cpu=\"124\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545329337
Jul 17 03:09:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:53.025Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", cpu=\"108\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545329337
Jul 17 03:09:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:53.038Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-virtual-net-veth36d3ae80.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689545325496
Jul 17 03:09:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:53.076Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_metadata_generation\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"traefik-64f55bb67d\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545324478
Jul 17 03:09:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:53.220Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_restart_policy\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"ipmi-exporter-6f8464dd94-ghpck\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", type=\"Always\", uid=\"ff360f8e-d8b6-4451-9e19-c901d1e7ae8c\"}" timestamp=1689545328950
Jul 17 03:09:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:53.267Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"msh-tsgx08\", asset_id=\"65\", condition=\"true\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-pushgateway-bd476bcf9-h4x26\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"f5340e71-ce09-4741-86dd-e1c191caf00d\"}" timestamp=1689545328950
Jul 17 03:09:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:53.281Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_labels\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"chart-values-tsg-traffic-engine-numa-0\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545328950
Jul 17 03:09:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:53.299Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"ipmi_up\", asset=\"msh-tsgx12\", asset_id=\"113\", collector=\"bmc\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"ipmi-exporter\"}" timestamp=1689545325070
Jul 17 03:09:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:53.315Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_packets_total\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"cadvisor\", interface=\"ens8f0np0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545311824
Jul 17 03:09:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:53.335Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx08\", asset_id=\"65\", cpu=\"87\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545328328
Jul 17 03:09:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:53.369Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_name_assign_type\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"vethf05dc1bb\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545328328
Jul 17 03:09:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:53.376Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_owner\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", owner_is_controller=\"true\", owner_kind=\"Deployment\", owner_name=\"configmap-reloader\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"configmap-reloader-6c55bbc49f\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545324478
Jul 17 03:09:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:53.411Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_type\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"chart-values-tsg-traffic-engine-numa-1\", service=\"prometheus-kube-state-metrics\", type=\"Opaque\"}" timestamp=1689545324478
Jul 17 03:09:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:53.443Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"tmpfs\", name=\"run-k3s-containerd-io.containerd.grpc.v1.cri-sandboxes-a4e62bf68e957def5fc02156788b02d200f644be8ea2151b7063b937542ce54e-shm.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689545325496
Jul 17 03:09:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:53.458Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-c6e5f260bd6464ec55b79eb8cfacfc9067038d44677275c90bd26045c55fe74f.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"scope\"}" timestamp=1689545325496
Jul 17 03:09:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:53.584Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2dpartuuid-abf62e28\\\\x2d6d2b\\\\x2d4943\\\\x2d8c7e\\\\x2dcd93eb619814.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689545325496
Jul 17 03:09:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:53.652Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", pod=\"tsg-traffic-engine-numa-0-firewall-665bf977fd-6ss4j\", project=\"TSG-OS\", project_id=\"7\", reason=\"NodeAffinity\", service=\"prometheus-kube-state-metrics\", uid=\"a188b741-4451-4393-a8d5-35ceaab41710\"}" timestamp=1689545328663
Jul 17 03:09:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:53.683Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_fifo_total\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth9066c4e8\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545323391
Jul 17 03:09:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:53.702Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_labels\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", job_name=\"helm-install-traefik\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545328950
Jul 17 03:09:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:53.718Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_ready\", asset=\"msh-tsgx08\", asset_id=\"65\", container=\"dashboard-metrics-scraper\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", pod=\"dashboard-metrics-scraper-5b94897dc9-7xj9b\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"df708fd2-fb21-4af8-958e-4ba1dc3e2b4a\"}" timestamp=1689545328950
Jul 17 03:09:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:53.732Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-virtual-net-vethebc85272.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689545332135
Jul 17 03:09:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:53.751Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_bytes_total\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"cadvisor\", interface=\"virtio_dign_s\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545311824
Jul 17 03:09:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:53.772Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"promhttp_metric_handler_requests_total\", asset=\"msh-tsgx08\", asset_id=\"65\", code=\"500\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545328328
Jul 17 03:09:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:53.788Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db192\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545330108
Jul 17 03:09:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:53.810Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx08\", asset_id=\"65\", cpu=\"1\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545328328
Jul 17 03:09:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:53.921Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2dpartuuid-1a7d5956\\\\x2d3bd3\\\\x2d4ca6\\\\x2db37d\\\\x2d6afa5ee5d032.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689545320884
Jul 17 03:09:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:53.946Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_metadata_resource_version\", asset=\"pcap-tsgx07\", asset_id=\"85\", configmap=\"coredns\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545322058
Jul 17 03:09:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:53.980Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cpu=\"65\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545319409
Jul 17 03:09:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:54.052Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_swap\", asset=\"msh-tsgx12\", asset_id=\"113\", container=\"exporter-merger\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545317721
Jul 17 03:09:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:54.085Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_name_assign_type\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens8f1np1\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545328328
Jul 17 03:09:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:54.154Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_max_usage_bytes\", asset=\"msh-tsgx08\", asset_id=\"65\", container=\"dign-server-web\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545308983
Jul 17 03:09:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:54.189Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_max_fds\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"k3s.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689545331179
Jul 17 03:09:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:54.218Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_bytes_total\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"cadvisor\", interface=\"ens8f1np1\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545316892
Jul 17 03:09:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:54.222Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx08\", asset_id=\"65\", cpu=\"120\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545328328
Jul 17 03:09:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:54.272Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db27\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545325184
Jul 17 03:09:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:54.499Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_ready\", asset=\"pcap-tsgx07\", asset_id=\"85\", container=\"helm\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-tsg-traffic-engine-numa-1-f7wwg\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"06fbf17e-ade1-4a2a-aaea-eb530d9bc31e\"}" timestamp=1689545322058
Jul 17 03:09:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:54.526Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_metadata_resource_version\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.tsg-traffic-engine-numa-0.v1\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545322058
Jul 17 03:09:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:54.702Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_drop_total\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"cnibr0\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545328328
Jul 17 03:09:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:54.716Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_completion_time\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-delete-tsg-traffic-engine-numa-1-pv6hh\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"ee9f4174-ab05-411d-a216-578ec25bab94\"}" timestamp=1689545328950
Jul 17 03:09:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:54.835Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_hwmon_chip_names\", asset=\"pcap-tsgx07\", asset_id=\"85\", chip=\"0000:60:03_1_0000:63:00_3\", chip_name=\"bnxt_en\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545319409
Jul 17 03:09:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:54.880Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545325184
Jul 17 03:09:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:54.902Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Unknown_Bit\", app_name=\"sapp4\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", vsysId=\"1\"}" timestamp=1689545321765
Jul 17 03:09:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:54.923Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_drop_total\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"virtio_dign_s\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545323391
Jul 17 03:09:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:55.508Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_reservation_limit_bytes\", asset=\"pcap-tsgx07\", asset_id=\"85\", container=\"dashboard-metrics-scraper\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545330608
Jul 17 03:09:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:55.522Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cpu=\"124\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545319409
Jul 17 03:09:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:55.540Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cpu=\"76\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545319409
Jul 17 03:09:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:55.561Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-80d17ee2\\\\x2d8132\\\\x2d4555\\\\x2da25c\\\\x2d5aba9a6428c8-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2dpplts.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689545320884
Jul 17 03:09:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:56.192Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"clickhouse_block_active_time_sda\", asset=\"msh-dellr740-olap001\", asset_id=\"11\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Clickhouse-msh-dellr740-olap001\", endpoint_id=\"71\", module=\"NC-Clickhouse\", module_id=\"29\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545335803
Jul 17 03:09:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:56.283Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_max_usage_bytes\", asset=\"pcap-tsgx03\", asset_id=\"73\", container=\"telegraf\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689545321536
Jul 17 03:09:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:56.294Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"58\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545325883
Jul 17 03:09:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:56.376Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_hwmon_temp_celsius\", asset=\"pcap-tsgx03\", asset_id=\"73\", chip=\"pci0000:00_0000:00:18_3\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", sensor=\"temp4\", service=\"prometheus-node-exporter\"}" timestamp=1689545325883
Jul 17 03:09:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:56.378Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_scrape_collector_success\", asset=\"pcap-tsgx03\", asset_id=\"73\", collector=\"zfs\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545325883
Jul 17 03:09:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:56.394Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"106\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545325883
Jul 17 03:09:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:56.588Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_heap_released_bytes\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545326475
Jul 17 03:09:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:56.621Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_created\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_endpoint=\"prometheus-kube-state-metrics\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545333389
Jul 17 03:09:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:56.637Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_complete\", asset=\"pcap-tsgx03\", asset_id=\"73\", condition=\"unknown\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", job_name=\"helm-install-prometheus\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545333389
Jul 17 03:09:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:56.981Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_local_link_integrity_errors_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"mlx5_3\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545325883
Jul 17 03:09:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:57.243Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", container=\"dashboard-metrics-scraper\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Sync\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545325115
Jul 17 03:09:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:57.270Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_packets_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"cadvisor\", interface=\"ens4f3np3\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545331184
Jul 17 03:09:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:57.290Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"63\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545325883
Jul 17 03:09:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:57.960Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", cpu=\"23\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"OLAP-Node-Exporter-pcap-dellr740-dt001\", endpoint_id=\"251\", mode=\"system\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545337557
Jul 17 03:09:58 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:58.092Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_status_failed\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", job_name=\"helm-install-traefik\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545333389
Jul 17 03:09:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:59.674Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pnp0-00:03-tty-ttyS0.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689545324879
Jul 17 03:09:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:59.683Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"msh-tsgx01\", asset_id=\"51\", container=\"coredns\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"stopped\"}" timestamp=1689545321977
Jul 17 03:09:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:59.702Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_namespace_status_phase\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", phase=\"Active\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545327425
Jul 17 03:09:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:59.706Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-machine-id-commit.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"service\"}" timestamp=1689545324879
Jul 17 03:09:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:59.714Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"115\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545329083
Jul 17 03:09:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:59.723Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas_available\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"metrics-server\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545327425
Jul 17 03:09:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:59.727Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"9\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545329083
Jul 17 03:09:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:59.742Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx01\", asset_id=\"51\", container=\"prometheus-server-configmap-reload\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Total\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545323755
Jul 17 03:09:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:59.760Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"snmptrapd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689545324879
Jul 17 03:09:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:59.764Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"36\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545329083
Jul 17 03:09:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:59.774Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"48\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545329083
Jul 17 03:09:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:59.778Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"26\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545329083
Jul 17 03:09:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:09:59.788Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_observed_generation\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-1-firewall-7f9c998578\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545327425
Jul 17 03:10:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:00.089Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db49\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545325071
Jul 17 03:10:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:00.103Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:20-0000:20:03.1-0000:21:00.1-infiniband-mlx5_3.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689545324879
Jul 17 03:10:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:00.459Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx01\", asset_id=\"51\", container=\"configmap-reloader\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Total\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545330862
Jul 17 03:10:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:00.483Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"msh-tsgx01\", asset_id=\"51\", container=\"lb-tcp-80\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"cadvisor\", failure_type=\"pgmajfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"hierarchy\"}" timestamp=1689545326846
Jul 17 03:10:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:00.514Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx01\", asset_id=\"51\", container=\"mrapm-device\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Total\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545323713
Jul 17 03:10:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:00.537Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"shm\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/56d6b27513b7a137f9b23d18e06c0181d594b6e5cfcd1d17d263f3a1b30370d5/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545329083
Jul 17 03:10:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:00.626Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link_mode\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth366ee37d\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545329083
Jul 17 03:10:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:00.656Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_compressed_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth358d74e7\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545329083
Jul 17 03:10:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:00.685Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2dlabel-TGS\\\\x2dOS\\\\x2dUPDATE.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689545324879
Jul 17 03:10:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:00.733Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-burstable-podb1c94976_83e8_41e4_85f6_2606e4291f65.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"slice\"}" timestamp=1689545324879
Jul 17 03:10:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:01.570Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx01\", asset_id=\"51\", container=\"prometheus-alertmanager\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Read\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545325749
Jul 17 03:10:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:01.656Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_spec_strategy_rollingupdate_max_unavailable\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"configmap-reloader\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545327425
Jul 17 03:10:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:01.673Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"local-path-provisioner-957fdf8bc-7gcxw\", project=\"TSG-OS\", project_id=\"7\", reason=\"Evicted\", service=\"prometheus-kube-state-metrics\", uid=\"dc0b66cf-ac45-4d06-8dbb-3494485cf1d5\"}" timestamp=1689545327425
Jul 17 03:10:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:01.716Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_reads_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/shm\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545332871
Jul 17 03:10:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:01.732Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_sockets\", asset=\"msh-tsgx01\", asset_id=\"51\", container=\"dign-client\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545329673
Jul 17 03:10:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:01.799Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"59\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545329083
Jul 17 03:10:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:01.823Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_daemonset_status_number_unavailable\", asset=\"msh-tsgx01\", asset_id=\"51\", daemonset=\"svclb-traefik-7cfeb2e8\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545327425
Jul 17 03:10:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:01.828Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_start_time_seconds\", asset=\"msh-tsgx01\", asset_id=\"51\", container=\"telegraf\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689545334593
Jul 17 03:10:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:01.851Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"45\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545329083
Jul 17 03:10:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:01.866Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_name_assign_type\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens7f0np0\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545329083
Jul 17 03:10:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:02.351Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:40-0000:40:01.1-0000:41:00.1-infiniband-mlx5_5.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689545328447
Jul 17 03:10:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:02.894Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-coredump@0.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689545328447
Jul 17 03:10:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:03.087Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-e8a136056802f7734c1413e221c6eab5e91b83fe23099cc9f3b4ffc6823badfb.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"scope\"}" timestamp=1689545328447
Jul 17 03:10:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:03.279Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_rapl_package_joules_total\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-dt001\", endpoint_id=\"225\", index=\"0\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545342988
Jul 17 03:10:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:03.458Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"net_conntrack_dialer_conn_established_total\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", dialer_name=\"33\", instance=\"10.10.20.159:10090\", type=\"global\"}" timestamp=1689545342990
Jul 17 03:10:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:03.744Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"prometheus_http_response_size_bytes_bucket\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", handler=\"/metrics\", instance=\"10.10.20.159:10090\", le=\"100000\", type=\"global\"}" timestamp=1689545342990
Jul 17 03:10:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:03.927Z caller=dedupe.go:112 component=remote level=info remote_name=52db6f url=http://10.10.10.159:10090/nz-agent/prometheus/proxy/api/v1/write msg="Dropped sample for series that was not explicitly dropped via relabelling" ref=4436650
Jul 17 03:10:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:04.000Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dracut-pre-udev.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"service\"}" timestamp=1689545328447
Jul 17 03:10:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:04.054Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"exporter-systemd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"service\"}" timestamp=1689545328447
Jul 17 03:10:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:04.070Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"msh-tsgx02\", asset_id=\"63\", cpu=\"23\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545338318
Jul 17 03:10:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:04.390Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"net_conntrack_dialer_conn_failed_total\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", dialer_name=\"451\", instance=\"10.10.20.159:10090\", reason=\"unknown\", type=\"global\"}" timestamp=1689545342990
Jul 17 03:10:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:04.415Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"prometheus_http_request_duration_seconds_bucket\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", handler=\"/-/healthy\", instance=\"10.10.20.159:10090\", le=\"+Inf\", type=\"global\"}" timestamp=1689545342990
Jul 17 03:10:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:04.725Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"net_conntrack_dialer_conn_failed_total\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", dialer_name=\"187\", instance=\"10.10.20.159:10090\", reason=\"refused\", type=\"global\"}" timestamp=1689545342990
Jul 17 03:10:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:06.691Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"twa-tsgx01\", asset_id=\"91\", condition=\"true\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-tsg-traffic-engine-numa-1-nzg5m\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"f36b39da-a7f4-44d5-a1b0-fc87a0caf7ea\"}" timestamp=1689545341227
Jul 17 03:10:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:06.873Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_bytes_total\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"cadvisor\", interface=\"ens6f0np0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545342540
Jul 17 03:10:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:06.889Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_errors_received_total\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_11\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545343910
Jul 17 03:10:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:06.893Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-netns-cni\\\\x2d59fae33c\\\\x2d2597\\\\x2dcaa9\\\\x2d84cb\\\\x2d7921afd8780c.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689545333582
Jul 17 03:10:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:07.048Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"emergency.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"idle\", type=\"service\"}" timestamp=1689545333582
Jul 17 03:10:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:07.107Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"rc-local.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"service\"}" timestamp=1689545333582
Jul 17 03:10:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:07.209Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_labels\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"kubernetes-dashboard-key-holder\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545341227
Jul 17 03:10:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:07.307Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", phase=\"Running\", pod=\"systemd-exporter-7f549d9485-sdp4t\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"3fcd6bf2-9ff2-4a22-a045-c43c0b1a8091\"}" timestamp=1689545341227
Jul 17 03:10:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:07.315Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_queue_length\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth444895a4\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545343910
Jul 17 03:10:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:07.355Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cpu=\"104\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545343910
Jul 17 03:10:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:07.598Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_drop_total\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth34c9090\", endpoint=\"NC-Spark-msh-dellr740-olap003\", endpoint_id=\"13\", module=\"NC-Spark\", module_id=\"47\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545346091
Jul 17 03:10:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:07.702Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_frequency_min_hertz\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", cpu=\"27\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Spark-msh-dellr740-olap003\", endpoint_id=\"13\", module=\"NC-Spark\", module_id=\"47\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545346091
Jul 17 03:10:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:08.120Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-udevd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"notify\", type=\"service\"}" timestamp=1689545333582
Jul 17 03:10:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:08.298Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_queue_length\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"virtio_dign_c\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545343910
Jul 17 03:10:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:08.317Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cpu=\"61\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545343910
Jul 17 03:10:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:08.331Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db220\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545338290
Jul 17 03:10:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:11.009Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_compactionsFailedCount\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"default\", project=\"OLAP\", project_id=\"3\", region=\"b80f77b3ae41c2c5dc46743c217a463a\", table=\"index_filename_default\", type=\"index\"}" timestamp=1689545348804
Jul 17 03:10:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:11.447Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_replicaid\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"knowledge_base_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"8f77301947b837805e9b5a6254c5a226\", table=\"index_time_knowledge_base_hos_bucket\", type=\"index\"}" timestamp=1689545348804
Jul 17 03:10:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:11.472Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_compactionsCompletedCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"session_record_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"593e714e0aaccde4836629880eb9cc77\", table=\"index_filename_session_record_hos_bucket\", type=\"index\"}" timestamp=1689545350147
Jul 17 03:10:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:11.487Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_LOG_metric_deleteTime_98th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545350147
Jul 17 03:10:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:11.493Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_getCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"knowledge_base_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"c95a264dbee52b8d15a6e132ecfd0800\", table=\"index_time_knowledge_base_hos_bucket\", type=\"index\"}" timestamp=1689545350147
Jul 17 03:10:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:11.536Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_CHILD_LINK_metric_scanSize_95th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545350147
Jul 17 03:10:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:11.939Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_2_metric_compactionOutputFileCount_75th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545350147
Jul 17 03:10:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:11.954Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_scaling_frequency_hertz\", asset=\"twa-dellr740-dt002\", asset_id=\"39\", cpu=\"25\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"NC-Gohangout-twa-dellr740-dt002\", endpoint_id=\"179\", module=\"DC-Gohangout\", module_id=\"35\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545351525
Jul 17 03:10:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:12.006Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_session_record_hos_bucket_metric_compactionOutputFileCount_75th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545350147
Jul 17 03:10:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:12.024Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_maxFlushQueueSize\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"knowledge_base_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"bf2a63dbae3b7c03de972a5d882a0894\", table=\"index_filename_knowledge_base_hos_bucket\", type=\"index\"}" timestamp=1689545350147
Jul 17 03:10:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:12.324Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_ThreadsBlocked\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"JvmMetrics\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545350147
Jul 17 03:10:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:12.474Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_flushesQueuedCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"8b01009986d540aa1408698fe7536c9a\", table=\"troubleshooting_hos_bucket\", type=\"data\"}" timestamp=1689545350147
Jul 17 03:10:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:12.510Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_replicaid\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"session_record_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"fe5edc011eaddb1eb1c7961140d3dbc5\", table=\"index_filename_session_record_hos_bucket\", type=\"index\"}" timestamp=1689545350147
Jul 17 03:10:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:12.529Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_MajorCompactionTime_min\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Server\"}" timestamp=1689545350147
Jul 17 03:10:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:12.884Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_MUTEX_metric_compactionInputFileCount_90th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545350147
Jul 17 03:10:13 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:13.033Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_incrementCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", namespace=\"SYSTEM\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"2f6b6350c5e74cb4f4b4cf9e7376455c\", table=\"CHILD_LINK\", type=\"data\"}" timestamp=1689545350147
Jul 17 03:10:13 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:13.046Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_incrementCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"proxy_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"2097b60eaa4ffea308d53e366e4242fb\", table=\"index_filename_proxy_hos_bucket\", type=\"index\"}" timestamp=1689545350147
Jul 17 03:10:13 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:13.060Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_4_metric_compactionOutputSize_75th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545350147
Jul 17 03:10:13 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:13.075Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_1_metric_compactionInputSize_75th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545350147
Jul 17 03:10:13 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:13.092Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"java_lang_Compilation_CompilationTimeMonitoringSupported\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545350147
Jul 17 03:10:13 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:13.180Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_buck_hash_sys_bytes\", asset=\"twa-dellr740-dt002\", asset_id=\"39\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"OLAP-Node-Exporter-twa-dellr740-dt002\", endpoint_id=\"259\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545352055
Jul 17 03:10:13 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:13.185Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_session_record_hos_bucket_metric_flushMemstoreSize_99th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545350147
Jul 17 03:10:13 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:13.888Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-dellr740-dt002\", asset_id=\"39\", cpu=\"38\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"OLAP-Node-Exporter-twa-dellr740-dt002\", endpoint_id=\"259\", mode=\"softirq\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545352055
Jul 17 03:10:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:15.288Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"plymouth-switch-root.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"oneshot\", type=\"service\"}" timestamp=1689545347046
Jul 17 03:10:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:15.672Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_queue_length\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth5c296dad\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545341230
Jul 17 03:10:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:15.755Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_annotations\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.cm-policy-local-cache.v1\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545348076
Jul 17 03:10:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:16.037Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_annotations\", asset=\"twa-tsgx07\", asset_id=\"103\", configmap=\"chart-content-tsg-traffic-engine-numa-1\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545348076
Jul 17 03:10:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:16.059Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_files\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"shm\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/1e88ab467eb5d433e3551ba834ec5d53423871458d51d9a6c91448ff517f5abf/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545341230
Jul 17 03:10:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:16.089Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_errs_total\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"eno34np1\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545341230
Jul 17 03:10:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:16.124Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_info\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"chart-values-traefik\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545348076
Jul 17 03:10:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:16.249Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Bit_Inbound\", app_name=\"sapp4\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", vsysId=\"1\"}" timestamp=1689545351690
Jul 17 03:10:17 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:17.214Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_disk_flush_requests_total\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"sda\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545341230
Jul 17 03:10:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:19.466Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-pod78246b9b_6c2f_4139_8e55_13e52335743d.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"slice\"}" timestamp=1689545344356
Jul 17 03:10:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:19.940Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_created\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_endpoint=\"kubernetes\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545353949
Jul 17 03:10:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:19.975Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"4\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545357612
Jul 17 03:10:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:20.779Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"unbound-anchor.timer\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"timer\"}" timestamp=1689545350657
Jul 17 03:10:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:20.797Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"tsg_master_opening\", app_name=\"tsg_master\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"traffic-engine\", line_name=\"fail\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", table_name=\"ctrl_stat\", vsysId=\"1\"}" timestamp=1689545353544
Jul 17 03:10:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:20.824Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_mapped_file\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689545351896
Jul 17 03:10:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:20.839Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_swap\", asset=\"twa-tsgx06\", asset_id=\"101\", container=\"dashboard-metrics-scraper\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545337096
Jul 17 03:10:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:20.904Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Unknown\", pod=\"metrics-server-54dc485875-lxknt\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"9d2e834d-aee4-4f71-86f6-55718dbb43e3\"}" timestamp=1689545353828
Jul 17 03:10:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:21.010Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"125\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545348263
Jul 17 03:10:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:21.028Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"scrape_duration_seconds\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_instance=\"localhost\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545352894
Jul 17 03:10:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:21.044Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_mtu_bytes\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth3417e171\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545348263
Jul 17 03:10:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:21.079Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"pcap-tsgx10\", asset_id=\"89\", condition=\"true\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-traefik-b7tft\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"f8334eab-55b4-4187-9700-4ad2897e2d91\"}" timestamp=1689545353828
Jul 17 03:10:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:21.090Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"configmap-reloader-6c55bbc49f-sdx4c\", project=\"TSG-OS\", project_id=\"7\", reason=\"NodeAffinity\", service=\"prometheus-kube-state-metrics\", uid=\"d69b2139-72e0-47ab-b29f-fa787b9e63b9\"}" timestamp=1689545353828
Jul 17 03:10:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:21.092Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-4cb5f0e8ce74a18b62361d9117cccdb52384a274068f68042f305e7447d47135-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689545350657
Jul 17 03:10:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:21.112Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"24\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545348263
Jul 17 03:10:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:21.124Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_errs_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"eno36np3\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545348263
Jul 17 03:10:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:21.132Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"100\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545348263
Jul 17 03:10:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:21.137Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db148\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545354840
Jul 17 03:10:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:21.153Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_heap_inuse_bytes\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545351998
Jul 17 03:10:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:21.172Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens4f2np2\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545357612
Jul 17 03:10:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:21.175Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_cache\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545352803
Jul 17 03:10:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:21.193Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_errors_total\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"cadvisor\", interface=\"eno1\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545350908
Jul 17 03:10:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:21.234Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"iscsi-onboot.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689545344356
Jul 17 03:10:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:21.243Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"60\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545348263
Jul 17 03:10:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:21.249Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failcnt\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689545342599
Jul 17 03:10:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:21.259Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"tsg-clixon-backend.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689545350657
Jul 17 03:10:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:21.271Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"pcap-tsgx10\", asset_id=\"89\", condition=\"unknown\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-ipmi-exporter-ntwmm\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"6d124c79-91fd-435c-a730-222ebe3b4194\"}" timestamp=1689545353828
Jul 17 03:10:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:21.284Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"107\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545348263
Jul 17 03:10:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:21.295Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-udev-trigger.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"service\"}" timestamp=1689545350657
Jul 17 03:10:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:21.393Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"rx_bits_total\", app=\"sapp4-0\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"nf_0_fw\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"mrzcpd-exporter\", serviceFunctionIndex=\"0\"}" timestamp=1689545354036
Jul 17 03:10:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:21.450Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_id\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth301b7957\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545348263
Jul 17 03:10:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:21.502Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"phy_tx_packets_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", dev=\"virtio_dign_c\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"mrzcpd-exporter\"}" timestamp=1689545354036
Jul 17 03:10:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:21.519Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_tracking_total_keys\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545354840
Jul 17 03:10:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:21.612Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", phase=\"Failed\", pod=\"prometheus-pushgateway-bd476bcf9-rlgwl\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"fe59f4a3-ab44-46a4-9b1d-4a02aedb6f14\"}" timestamp=1689545353949
Jul 17 03:10:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:21.625Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Bit_Outbound\", app_name=\"sapp4\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", vsysId=\"1\"}" timestamp=1689545353544
Jul 17 03:10:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:21.628Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", cpu=\"8\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"OLAP-Node-Exporter-twa-dellr740-dt001\", endpoint_id=\"257\", mode=\"nice\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545361405
Jul 17 03:10:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:21.816Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"pcap-tsgx10\", asset_id=\"89\", condition=\"false\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"svclb-traefik-742ef4d3-452pc\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"88ae83c1-0ebf-40d5-bed7-6f1b41109c68\"}" timestamp=1689545353828
Jul 17 03:10:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:21.930Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"scrape_samples_post_metric_relabeling\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_instance=\"10.10.10.170:9008\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689545350657
Jul 17 03:10:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:21.956Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kmod-static-nodes.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"service\"}" timestamp=1689545350657
Jul 17 03:10:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:21.991Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.grpc.v1.cri-sandboxes-4cb5f0e8ce74a18b62361d9117cccdb52384a274068f68042f305e7447d47135-shm.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689545350657
Jul 17 03:10:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:21.998Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"pcap-tsgx10\", asset_id=\"89\", condition=\"true\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"local-path-provisioner-957fdf8bc-6hrfp\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"03ed11d6-2d8c-4174-885e-b558bd212ab7\"}" timestamp=1689545353828
Jul 17 03:10:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:22.525Z caller=dedupe.go:112 component=remote level=error remote_name=52db6f url=http://10.10.10.159:10090/nz-agent/prometheus/proxy/api/v1/write msg="non-recoverable error" count=1000 exemplarCount=0 err="server returned HTTP status 400 : out of order sample"
Jul 17 03:10:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:22.787Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-dellr740-olap004\", asset_id=\"17\", cpu=\"20\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Spark-msh-dellr740-olap004\", endpoint_id=\"15\", mode=\"user\", module=\"NC-Spark\", module_id=\"47\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545362253
Jul 17 03:10:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:23.565Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_drop_total\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"lo\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545354790
Jul 17 03:10:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:23.580Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_nf_conntrack_entries_limit\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545354790
Jul 17 03:10:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:23.627Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"96\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545354790
Jul 17 03:10:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:23.675Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx09\", asset_id=\"55\", cpu=\"22\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545354790
Jul 17 03:10:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:24.046Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_commands_total\", asset=\"msh-tsgx09\", asset_id=\"55\", cmd=\"latency\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545357019
Jul 17 03:10:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:24.104Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth0c6b6d72\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545354790
Jul 17 03:10:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:24.122Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx09\", asset_id=\"55\", cpu=\"42\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545354790
Jul 17 03:10:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:24.162Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_defrag_key_misses\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545357019
Jul 17 03:10:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:24.163Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_address_assign_type\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"virtio_dign_s\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545354790
Jul 17 03:10:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:24.214Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_flags\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens4f3np3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545354790
Jul 17 03:10:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:24.233Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"blk-availability.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"oneshot\", type=\"service\"}" timestamp=1689545358416
Jul 17 03:10:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:24.234Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db70\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545357019
Jul 17 03:10:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:24.247Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sys_mem_max\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Druid-msh-dellr740-olap003\", endpoint_id=\"383\", host=\"192.168.20.223:8083\", module=\"NC-Druid\", module_id=\"31\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", service=\"historical\", version=\"0.18.1\"}" timestamp=1689545363359
Jul 17 03:10:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:24.578Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"79\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545360160
Jul 17 03:10:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:24.622Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_restart_policy\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-configmap-reloader-zk7vg\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", type=\"OnFailure\", uid=\"d5afaa6d-3d18-460e-a5d3-68d88dfe8aed\"}" timestamp=1689545351122
Jul 17 03:10:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:24.692Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_system_seconds_total\", asset=\"msh-tsgx09\", asset_id=\"55\", container=\"dashboard-metrics-scraper\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545356539
Jul 17 03:10:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:24.893Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_bytes_total\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", interface=\"bond0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545358144
Jul 17 03:10:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:24.952Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_commands_rejected_calls_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cmd=\"unlink\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545361795
Jul 17 03:10:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:24.974Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-eno34np1.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689545351102
Jul 17 03:10:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:24.980Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_address_assign_type\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"vethc31b7dd7\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545360160
Jul 17 03:10:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:25.384Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx09\", asset_id=\"55\", container=\"configmap-reloader\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Total\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545340148
Jul 17 03:10:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:25.476Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"pcap-kvm-cm001\", asset_id=\"45\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db144\", endpoint=\"CM-Branch-Redis-pcap-kvm-cm001\", endpoint_id=\"321\", host=\"192.168.10.157\", module=\"CM-Branch-Redis\", module_id=\"73\", nz_agent_id=\"3\", parent_asset=\"pcap-dellr740-pve001\", parent_asset_id=\"43\", project=\"Central-Management\", project_id=\"5\"}" timestamp=1689545365366
Jul 17 03:10:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:25.510Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"pcap-tsgx12\", asset_id=\"117\", condition=\"false\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"traefik-64f55bb67d-xrfgg\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"fd6dc4c2-5a78-40f9-8373-71b885a791c5\"}" timestamp=1689545351122
Jul 17 03:10:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:26.113Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"init.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"scope\"}" timestamp=1689545353907
Jul 17 03:10:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:26.165Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-e9990f8b\\\\x2d7450\\\\x2d47a8\\\\x2d867f\\\\x2d7387f0ac7e25-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2dplsfl.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689545351102
Jul 17 03:10:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:26.530Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_ports\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_endpoint=\"prometheus-server\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", port_name=\"http\", port_number=\"9090\", port_protocol=\"TCP\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545351122
Jul 17 03:10:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:26.623Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"81\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545360160
Jul 17 03:10:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:26.841Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_info\", asset=\"pcap-tsgx12\", asset_id=\"117\", created_by_kind=\"ReplicaSet\", created_by_name=\"prometheus-alertmanager-6df7f457b6\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", host_ip=\"10.10.10.172\", host_network=\"false\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", node=\"localhost\", nz_agent_id=\"3\", pod=\"prometheus-alertmanager-6df7f457b6-2h45l\", pod_ip=\"10.42.0.32\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"b60f186f-e535-4016-adab-bd4c742131b8\"}" timestamp=1689545351122
Jul 17 03:10:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:26.875Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_compressed_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens7f0np0\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545360160
Jul 17 03:10:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:27.185Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_start_time_seconds\", asset=\"twa-tsgx09\", asset_id=\"107\", container=\"telegraf\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689545365490
Jul 17 03:10:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:27.221Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_created\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"metrics-server\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545358744
Jul 17 03:10:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:27.248Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545355810
Jul 17 03:10:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:27.302Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Udp_Bypass_Bit\", app_name=\"sapp4\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", vsysId=\"1\"}" timestamp=1689545351010
Jul 17 03:10:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:27.421Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"51\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545355810
Jul 17 03:10:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:27.616Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_owner\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", job_name=\"helm-install-traefik-crd\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", owner_is_controller=\"<none>\", owner_kind=\"<none>\", owner_name=\"<none>\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545358744
Jul 17 03:10:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:28.139Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_created\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"kubernetes-dashboard-csrf\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545355682
Jul 17 03:10:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:28.162Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"52\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545362882
Jul 17 03:10:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:28.194Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"clickhouse_block_discard_ops_sdc\", asset=\"msh-dellr740-dt003\", asset_id=\"9\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Clickhouse-msh-dellr740-dt003\", endpoint_id=\"75\", module=\"NC-Clickhouse\", module_id=\"29\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545366499
Jul 17 03:10:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:28.207Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_swap_limit_bytes\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"exporter\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545364841
Jul 17 03:10:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:28.252Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"72\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545362882
Jul 17 03:10:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:28.261Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_metadata_resource_version\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.systemd-exporter.v1\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545355682
Jul 17 03:10:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:28.274Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_errs_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth68516c90\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545362882
Jul 17 03:10:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:28.277Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"nsfs\", name=\"run-netns-cni\\\\x2d7d32de96\\\\x2d71e6\\\\x2d5bc1\\\\x2daf72\\\\x2dd6963e91fb2c.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689545354114
Jul 17 03:10:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:28.283Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_id\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth3ea9a2cc\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545362882
Jul 17 03:10:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:28.381Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"65\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545355810
Jul 17 03:10:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:28.418Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"116\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545355810
Jul 17 03:10:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:28.441Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"97\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545355810
Jul 17 03:10:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:28.459Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"17\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545355810
Jul 17 03:10:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:28.475Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_multicast_total\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethab524db8\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545355810
Jul 17 03:10:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:28.538Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_carrier_total\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"eno35np2\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545355810
Jul 17 03:10:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:28.559Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_uname_info\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", domainname=\"(none)\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", machine=\"x86_64\", module=\"TSG-OS\", module_id=\"77\", nodename=\"twa-tsgx09\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", release=\"5.17.15-1.el8.x86_64\", service=\"prometheus-node-exporter\", sysname=\"Linux\", version=\"#1 SMP PREEMPT Wed Jun 15 02:07:24 UTC 2022\"}" timestamp=1689545355810
Jul 17 03:10:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:28.578Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Ipv6_Bit\", app_name=\"sapp4\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", vsysId=\"1\"}" timestamp=1689545355421
Jul 17 03:10:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:28.711Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_waiting\", asset=\"twa-tsgx09\", asset_id=\"107\", container=\"prometheus-alertmanager\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-alertmanager-6df7f457b6-77pk9\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"2edb1c6e-4f58-4c31-b8c5-28dfc8f91f4d\"}" timestamp=1689545358744
Jul 17 03:10:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:28.853Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"119\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545355810
Jul 17 03:10:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:28.870Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_queue_length\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"br_dign_c\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545355810
Jul 17 03:10:29 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:29.024Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_daemonset_metadata_generation\", asset=\"msh-tsgx10\", asset_id=\"69\", daemonset=\"prometheus-node-exporter\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545355682
Jul 17 03:10:29 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:29.229Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_files\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda4\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", fstype=\"ext4\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545362882
Jul 17 03:10:29 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:29.344Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"selinux-autorelabel-mark.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689545354114
Jul 17 03:10:29 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:29.350Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_waiting\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"dign-server-dns\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"dign-server-pf96k\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"75dc4160-da0a-4767-823a-227b8dd303c1\"}" timestamp=1689545355682
Jul 17 03:10:29 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:29.380Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"80\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545362882
Jul 17 03:10:29 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:29.502Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"12\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545362882
Jul 17 03:10:29 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:29.559Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2dpartlabel-TSG\\\\x2dOS\\\\x2dDATA.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689545354114
Jul 17 03:10:29 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:29.656Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_info\", address=\"08:c0:eb:59:d3:28\", asset=\"msh-tsgx10\", asset_id=\"69\", broadcast=\"ff:ff:ff:ff:ff:ff\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens6f0np0\", duplex=\"unknown\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operstate=\"down\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545362882
Jul 17 03:10:29 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:29.774Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_id\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth875ea1eb\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545362882
Jul 17 03:10:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:31.084Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-br_dign_c.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689545361015
Jul 17 03:10:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:31.126Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_memory_used_scripts_bytes\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545370141
Jul 17 03:10:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:31.158Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"tsg_master_success_log_s\", app_name=\"tsg_master\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"traffic-engine\", line_name=\"VOIP-RECORD\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", table_name=\"sendlog\", vsysId=\"1\"}" timestamp=1689545369611
Jul 17 03:10:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:31.186Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"62\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545360939
Jul 17 03:10:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:31.229Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_address_assign_type\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"virtio_dign_s\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545360939
Jul 17 03:10:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:31.350Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"msh-tsgx05\", asset_id=\"67\", condition=\"true\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"mrzcpd-exporter-5cf7f967cc-sjmvx\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"35481fc8-2e01-4ba2-8edd-0511e97d2f24\"}" timestamp=1689545364068
Jul 17 03:10:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:31.378Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-d8012c00\\\\x2daaec\\\\x2d4f56\\\\x2da35b\\\\x2dad6f4d1ef590-volume\\\\x2dsubpaths-telegraf-telegraf\\\\x2dsecurity-0.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689545361015
Jul 17 03:10:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:31.400Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_errs_total\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth66897ea4\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545360939
Jul 17 03:10:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:31.450Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_errs_total\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"cnibr0\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545360939
Jul 17 03:10:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:31.454Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"nvmefc-boot-connections.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"oneshot\", type=\"service\"}" timestamp=1689545361015
Jul 17 03:10:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:31.485Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dracut-pre-trigger.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689545361015
Jul 17 03:10:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:31.496Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_scrape_collector_duration_seconds\", asset=\"msh-tsgx05\", asset_id=\"67\", collector=\"netdev\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545360939
Jul 17 03:10:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:31.543Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cpupower.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689545361015
Jul 17 03:10:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:31.569Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"lm_sensors.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689545361015
Jul 17 03:10:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:31.587Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_labels\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.systemd-exporter.v1\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545370436
Jul 17 03:10:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:31.590Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-16b0c54f99bc831e5c2f40ec71018ed8402840a09fc9afa8e373fbc9e8816ba5-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689545361015
Jul 17 03:10:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:31.609Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_mtu_bytes\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth1d757172\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545360939
Jul 17 03:10:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:31.632Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_repl_backlog_is_active\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545370141
Jul 17 03:10:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:31.658Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_compressed_total\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth39378f16\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545359093
Jul 17 03:10:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:31.817Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_info\", asset=\"msh-tsgx05\", asset_id=\"67\", configmap=\"shaping-tsg-traffic-engine-numa-0\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545364068
Jul 17 03:10:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:31.887Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethf7cbe37a\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545359093
Jul 17 03:10:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:32.062Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_created\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-prometheus-s7vqp\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"8b1286e7-fc1f-4e1e-a7a7-215916c4185e\"}" timestamp=1689545359276
Jul 17 03:10:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:32.082Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"127\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545360791
Jul 17 03:10:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:32.086Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", pod=\"tsg-traffic-engine-numa-1-firewall-6d869d4b4c-7pv8f\", project=\"TSG-OS\", project_id=\"7\", reason=\"UnexpectedAdmissionError\", service=\"prometheus-kube-state-metrics\", uid=\"d8012c00-aaec-4f56-a35b-ad6f4d1ef590\"}" timestamp=1689545364068
Jul 17 03:10:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:32.096Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-podabf27d03_a067_448d_81a8_11df769a2208.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"slice\"}" timestamp=1689545362644
Jul 17 03:10:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:32.102Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"75\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545360939
Jul 17 03:10:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:32.120Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db103\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545370141
Jul 17 03:10:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:32.136Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-podadf25328_be5b_4a8f_99a7_ca39872df08e.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"slice\"}" timestamp=1689545361015
Jul 17 03:10:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:32.285Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-rdma-devices-mlx5_10:1-umad.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689545361015
Jul 17 03:10:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:32.382Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_ready\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"mrapm-device\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"mrzcpd-exporter-5cf7f967cc-hrwcm\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"de0f6c6e-3a5e-493f-bc37-51ac5bb8c9a3\"}" timestamp=1689545359276
Jul 17 03:10:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:32.403Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"8\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545360791
Jul 17 03:10:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:32.422Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"21\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545360791
Jul 17 03:10:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:32.427Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:20-0000:20:03.1-0000:21:00.0-infiniband-mlx5_2.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689545361015
Jul 17 03:10:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:32.454Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dracut-shutdown-onfailure.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"service\"}" timestamp=1689545362644
Jul 17 03:10:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:32.457Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"29\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545360605
Jul 17 03:10:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:32.467Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:a0-0000:a0:03.1-0000:a1:00.0-net-ens6f0np0.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689545362644
Jul 17 03:10:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:32.495Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"44\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545360605
Jul 17 03:10:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:32.564Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"108\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545360791
Jul 17 03:10:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:32.609Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_avg_ttl_seconds\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db0\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545366438
Jul 17 03:10:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:32.622Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_annotations\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"kube-multus-ds-qhs84\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"46949245-283c-4ce8-81e4-518f0d540de5\"}" timestamp=1689545364068
Jul 17 03:10:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:32.636Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"20\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545360939
Jul 17 03:10:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:32.654Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db246\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545366438
Jul 17 03:10:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:32.659Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"15\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545360791
Jul 17 03:10:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:32.669Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_system_seconds_total\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"monit-stream\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545358559
Jul 17 03:10:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:32.674Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-d92ddf9eb599bea6ea57485be448ea0a442db646b5d3fb2c1b248b41ddd3d2f7.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"scope\"}" timestamp=1689545362644
Jul 17 03:10:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:32.685Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Udp_Link_S2C\", app_name=\"sapp4\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", vsysId=\"1\"}" timestamp=1689545360718
Jul 17 03:10:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:32.687Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_processes\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"telegraf-security\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689545353172
Jul 17 03:10:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:32.693Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_annotations\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"configmap-reloader\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545359276
Jul 17 03:10:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:32.703Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545360939
Jul 17 03:10:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:32.703Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_complete\", asset=\"msh-tsgx11\", asset_id=\"111\", condition=\"unknown\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", job_name=\"helm-install-mrzcpd-exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545359276
Jul 17 03:10:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:32.705Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_address_not_ready\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_endpoint=\"cm-policy-local-cache-redis-master\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545370436
Jul 17 03:10:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:32.726Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_running\", asset=\"twa-tsgx03\", asset_id=\"95\", container=\"kube-state-metrics\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-kube-state-metrics-6b768c7bf8-8z25n\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"0f76af51-4cb4-40c5-8ff5-c445f8b7b608\"}" timestamp=1689545370436
Jul 17 03:10:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:32.746Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_multicast_total\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth2cff15d8\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545360939
Jul 17 03:10:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:32.768Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545360605
Jul 17 03:10:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:32.774Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"var-lib-kubelet-pods-a27de96d\\\\x2db0ab\\\\x2d4183\\\\x2d97e5\\\\x2da8990b41de63-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2drjng9.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689545362644
Jul 17 03:10:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:32.787Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_system_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", container=\"redis\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545350746
Jul 17 03:10:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:32.791Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_stack_inuse_bytes\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545369260
Jul 17 03:10:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:32.791Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failcnt\", asset=\"twa-tsgx05\", asset_id=\"99\", container=\"lb-tcp-443\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545348520
Jul 17 03:10:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:32.807Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_ips\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", ip=\"10.42.0.78\", ip_family=\"4\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"cm-policy-local-cache-redis-master-69967b5879-vlgl8\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"e8ef10b6-3745-4720-b3f4-f80080df3d45\"}" timestamp=1689545359894
Jul 17 03:10:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:32.822Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-ceca645e6975d39f0681d6e1a0265568d8df6702f42db7fe01963ad704c12709.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"scope\"}" timestamp=1689545367691
Jul 17 03:10:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:32.822Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_packets_total\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"br_dign_c\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545360939
Jul 17 03:10:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:32.826Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_metadata_generation\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"mrzcpd-exporter-5cf7f967cc\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545359894
Jul 17 03:10:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:32.835Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-99e7b7f0\\\\x2d307f\\\\x2d4274\\\\x2da670\\\\x2dfaf429f178f7-volume\\\\x2dsubpaths-config\\\\x2dvolume-telegraf-2.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689545361015
Jul 17 03:10:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:32.840Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dracut-shutdown-onfailure.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"oneshot\", type=\"service\"}" timestamp=1689545361015
Jul 17 03:10:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:32.854Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_cpu_period\", asset=\"twa-tsgx05\", asset_id=\"99\", container=\"prometheus-server-configmap-reload\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545358535
Jul 17 03:10:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:32.893Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545360605
Jul 17 03:10:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:32.909Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_memory_used_dataset_bytes\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545369260
Jul 17 03:10:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:32.910Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_mtu_bytes\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth6bfabd57\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545360791
Jul 17 03:10:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:32.922Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_virtual_memory_bytes\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-udevd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689545367691
Jul 17 03:10:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:32.937Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-ea90f3fa91f6a24043b48cb12249048b2833a6d427e8699e191128ec48f61c39-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689545367691
Jul 17 03:10:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:33.037Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"108\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545360605
Jul 17 03:10:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:33.050Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"87\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545360605
Jul 17 03:10:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:33.072Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db249\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545369260
Jul 17 03:10:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:33.084Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"86\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545360605
Jul 17 03:10:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:33.108Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"83\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545360605
Jul 17 03:10:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:33.167Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_max_fds\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"packagekit.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689545367691
Jul 17 03:10:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:33.175Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", phase=\"Succeeded\", pod=\"dashboard-metrics-scraper-5b94897dc9-vgszb\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"4d0583db-52e0-420b-838b-92b5b61d0a0b\"}" timestamp=1689545370436
Jul 17 03:10:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:33.264Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"15\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545360605
Jul 17 03:10:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:33.276Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"8\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545360605
Jul 17 03:10:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:33.290Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_mtu_bytes\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens4f2np2\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545360791
Jul 17 03:10:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:33.291Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-7398125dcac036e9467a5e64699d8816e5ffa907fc39919e5cadea0af235e691.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"scope\"}" timestamp=1689545367691
Jul 17 03:10:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:34.141Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-e8ef10b6\\\\x2d3745\\\\x2d4720\\\\x2db3f4\\\\x2df80080df3d45-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2dhbcql.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689545367691
Jul 17 03:10:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:34.405Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_labels\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-traefik-crd-bf9k4\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"39ad39af-b9c7-4eb1-a7db-349fcd19b742\"}" timestamp=1689545359894
Jul 17 03:10:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:34.433Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"twa-tsgx05\", asset_id=\"99\", condition=\"true\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-ipmi-exporter-qg4pr\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"6d488923-85e6-4a2b-a06b-4a00d69292b7\"}" timestamp=1689545359894
Jul 17 03:10:38 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:38.657Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_QuotaObserverChoreTime_median\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-Hmaster\", endpoint_id=\"341\", module=\"DC-HBase\", module_id=\"3\", name=\"Master\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Quotas\"}" timestamp=1689545376971
Jul 17 03:10:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:40.415Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_packets_dropped_total\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"cadvisor\", interface=\"ens3f1np1\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545368094
Jul 17 03:10:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:40.436Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_reads_total\", asset=\"twa-tsgx04\", asset_id=\"97\", container=\"kubernetes-dashboard\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545361732
Jul 17 03:10:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:40.467Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_daemonset_status_number_available\", asset=\"twa-tsgx04\", asset_id=\"97\", daemonset=\"dign-server\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545380034
Jul 17 03:10:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:40.505Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled_time\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-prometheus-vzwtx\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"1187ac6c-e3c7-4185-8a91-8bb4228b3b51\"}" timestamp=1689545380034
Jul 17 03:10:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:40.524Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_observed_generation\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"prometheus-alertmanager\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545380034
Jul 17 03:10:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:40.564Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_persistentvolumeclaim_status_phase\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", persistentvolumeclaim=\"prometheus-server\", phase=\"Bound\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545380034
Jul 17 03:10:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:40.591Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-netns-cni\\\\x2dfcbadcb6\\\\x2d8695\\\\x2d802f\\\\x2d2160\\\\x2d018e2f50755e.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689545375764
Jul 17 03:10:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:40.602Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_created\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"prometheus-server\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545380034
Jul 17 03:10:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:40.644Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"twa-tsgx04\", asset_id=\"97\", condition=\"true\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", pod=\"tsg-traffic-engine-numa-1-firewall-7bff8557d6-cnnft\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"9ad11045-e0bd-4481-9c2d-4c81a67475ee\"}" timestamp=1689545380034
Jul 17 03:10:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:40.670Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"twa-tsgx04\", asset_id=\"97\", condition=\"unknown\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"dign-client-dqh82\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"bdc36e8b-adec-435b-9e18-b67235cb2206\"}" timestamp=1689545380034
Jul 17 03:10:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:40.699Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"nvmefc-boot-connections.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"service\"}" timestamp=1689545375764
Jul 17 03:10:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:40.766Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-7cf4ec7701edc5cd7881d4bb89874d2cc68cb76eec61cf77630178f5a2fb2810.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"scope\"}" timestamp=1689545375764
Jul 17 03:10:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:40.841Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx04\", asset_id=\"97\", container=\"firewall\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Discard\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689545354650
Jul 17 03:10:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:40.875Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"twa-tsgx04\", asset_id=\"97\", container=\"traefik\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"cadvisor\", failure_type=\"pgmajfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"container\"}" timestamp=1689545368668
Jul 17 03:10:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:40.941Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_info\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", job_name=\"helm-install-traefik\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545380034
Jul 17 03:10:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:41.178Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_tasks_current\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-tmpfiles-setup.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689545375764
Jul 17 03:10:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:41.239Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_packets_total\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth2f178825\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545371647
Jul 17 03:10:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:41.276Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_carrier_total\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"eno33np0\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545371647
Jul 17 03:10:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:41.297Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx04\", asset_id=\"97\", container=\"dign-client\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Sync\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545363723
Jul 17 03:10:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:41.326Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_colls_total\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens4f0np0\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545371647
Jul 17 03:10:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:41.330Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_multicast_total\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"lo\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545371647
Jul 17 03:10:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:41.347Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_daemonset_status_observed_generation\", asset=\"twa-tsgx04\", asset_id=\"97\", daemonset=\"prometheus-node-exporter\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545380034
Jul 17 03:10:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:41.354Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"75\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545371647
Jul 17 03:10:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:41.383Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_resource_requests\", asset=\"twa-tsgx04\", asset_id=\"97\", container=\"metrics-server\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", node=\"localhost\", nz_agent_id=\"3\", pod=\"metrics-server-54dc485875-lkr7j\", project=\"TSG-OS\", project_id=\"7\", resource=\"memory\", service=\"prometheus-kube-state-metrics\", uid=\"4f4807f1-9973-4c05-9357-9438c02bd87b\", unit=\"byte\"}" timestamp=1689545380034
Jul 17 03:10:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:41.389Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-887f4dca682510d424145cd93eba067f462923cd0aa0a490b5f5d213b75596ef.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"scope\"}" timestamp=1689545375764
Jul 17 03:10:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:41.419Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx04\", asset_id=\"97\", container=\"dign-server-dns\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Read\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545350956
Jul 17 03:10:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:41.448Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.grpc.v1.cri-sandboxes-54aade828e34c3fbae585b4f55ad41e7ca9301fec8550fd1548625ab87205ed9-shm.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689545375764
Jul 17 03:10:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:41.450Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"123\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545371647
Jul 17 03:10:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:41.465Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx04\", asset_id=\"97\", container=\"firewall\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Read\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689545355866
Jul 17 03:10:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:41.689Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_storeFileSize\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"e53710e45d50cc3bc5c0a04bd6bbbfd5\", table=\"firewall_hos_bucket\", type=\"data\"}" timestamp=1689545379756
Jul 17 03:10:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:41.764Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_labels\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"mrzcpd-exporter-5cf7f967cc\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545380034
Jul 17 03:10:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:41.797Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas_ready\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"tsg-traffic-engine-numa-0-firewall\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545380034
Jul 17 03:10:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:41.838Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_replicaid\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"a8203691ca1400dbe5974df9484d373d\", table=\"default\", type=\"data\"}" timestamp=1689545379756
Jul 17 03:10:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:41.857Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_getCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"session_record_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"587346287ff7fe7466c7378edf2e47ec\", table=\"index_filename_session_record_hos_bucket\", type=\"index\"}" timestamp=1689545379756
Jul 17 03:10:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:42.094Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_packets_dropped_total\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"cadvisor\", interface=\"virtio_dign_s\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545368094
Jul 17 03:10:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:42.110Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_timer_last_trigger_seconds\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sysstat-collect.timer\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689545375764
Jul 17 03:10:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:42.162Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled_time\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-tsg-traffic-engine-numa-0-qwtq5\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"8bcaaf2d-628e-4f10-872c-7a32ecf2946e\"}" timestamp=1689545380034
Jul 17 03:10:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:42.182Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_storeFileSize\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"knowledge_base_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"04d699da099bd2e5815e410973cf631a\", table=\"index_time_knowledge_base_hos_bucket\", type=\"index\"}" timestamp=1689545379756
Jul 17 03:10:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:42.203Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_CHILD_LINK_metric_putTime_num_ops\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545379756
Jul 17 03:10:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:42.221Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_maxFlushQueueSize\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"troubleshooting_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"05721f17e194285e23d3ebeca06fcb49\", table=\"index_time_troubleshooting_hos_bucket\", type=\"index\"}" timestamp=1689545379756
Jul 17 03:10:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:42.248Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"twa-tsgx04\", asset_id=\"97\", container=\"lb-tcp-80\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"running\"}" timestamp=1689545364418
Jul 17 03:10:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:42.295Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_owner\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", owner_is_controller=\"true\", owner_kind=\"Deployment\", owner_name=\"tsg-traffic-engine-numa-1-firewall\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-1-firewall-7f7564848f\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545380034
Jul 17 03:10:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:42.447Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_system_table_bucket_meta_metric_majorCompactionTime_99th_percentile\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545379756
Jul 17 03:10:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:42.497Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", container=\"configmap-reloader\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Sync\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545376521
Jul 17 03:10:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:42.550Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_status_completion_time\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", job_name=\"helm-install-prometheus\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545375154
Jul 17 03:10:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:42.572Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_created\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"systemd-exporter-7f549d9485-7kvgn\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"27368c72-efff-405c-9fe0-1a20af809631\"}" timestamp=1689545375154
Jul 17 03:10:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:42.663Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_avail_bytes\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda2\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", fstype=\"ext4\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/mnt/onie-boot\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545376242
Jul 17 03:10:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:42.678Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"tmpfs\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/data/var/lib/kubelet/pods/98daa3ea-45f0-41c2-be94-512fb3e2a016/volumes/kubernetes.io~projected/kube-api-access-7768b\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545376242
Jul 17 03:10:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:42.755Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_timex_pps_jitter_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545376242
Jul 17 03:10:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:42.780Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", container=\"configmap-reloader\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Write\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545376521
Jul 17 03:10:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:42.896Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_fifo_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens4f0np0\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545376242
Jul 17 03:10:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:42.964Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", cpu=\"16\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545376242
Jul 17 03:10:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:42.976Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_link_error_recovery_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"mlx5_2\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545376242
Jul 17 03:10:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:43.269Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_packets_dropped_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"cadvisor\", interface=\"virtio_dign_c\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545378795
Jul 17 03:10:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:43.336Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Read\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689545376308
Jul 17 03:10:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:43.339Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"ext4\", name=\"var-lib-kubelet-pods-a38bc0a0\\\\x2df54d\\\\x2d4548\\\\x2db0f1\\\\x2d6472f0282ca8-volume\\\\x2dsubpaths-config\\\\x2dvolume-telegraf-2.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689545380671
Jul 17 03:10:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:43.498Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_namespace_status_phase\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"consul\", nz_agent_id=\"3\", phase=\"Active\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545375154
Jul 17 03:10:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:43.510Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", cpu=\"102\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545376242
Jul 17 03:10:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:43.649Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", container=\"kube-multus\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Write\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545372920
Jul 17 03:10:48 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:48.721Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"java_lang_MemoryPool_PeakUsage_used\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"Metaspace\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545387438
Jul 17 03:10:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:49.001Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_hlogFileSize\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Server\"}" timestamp=1689545387438
Jul 17 03:10:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:49.001Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"java_lang_MemoryPool_CollectionUsage_init\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"Par Eden Space\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545387438
Jul 17 03:10:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:49.060Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_2_metric_compactionTime_mean\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545387438
Jul 17 03:10:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:49.148Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_maxStoreFileAge\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"knowledge_base_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"d2f78d7225dc730bfcd9893f9f48ed39\", table=\"index_time_knowledge_base_hos_bucket\", type=\"index\"}" timestamp=1689545387438
Jul 17 03:10:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:49.232Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_readRequestCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", namespace=\"SYSTEM\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"a289e6c6c4fb50405ce3745629bf659b\", table=\"LOG\", type=\"data\"}" timestamp=1689545387438
Jul 17 03:10:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:49.242Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_2_metric_compactionOutputFileCount_90th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545387438
Jul 17 03:10:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:49.274Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_3_metric_putBatchTime_99_9th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545387438
Jul 17 03:10:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:49.287Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_2_metric_majorCompactionOutputFileCount_98th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545387438
Jul 17 03:10:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:49.287Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"java_lang_Threading_CurrentThreadCpuTimeSupported\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545387438
Jul 17 03:10:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:49.298Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_1_metric_flushOutputSize_90th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545387438
Jul 17 03:10:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:49.322Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_numBytesCompactedCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"knowledge_base_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"d4e72b6e91a9b75ed312cff599c6a2fa\", table=\"index_partfile_knowledge_base_hos_bucket\", type=\"index\"}" timestamp=1689545387438
Jul 17 03:10:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:49.363Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_1_metric_flushMemstoreSize_99th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545387438
Jul 17 03:10:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:49.499Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_limit_bytes\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"kubernetes-dashboard\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545379050
Jul 17 03:10:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:49.511Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx10\", asset_id=\"109\", cpu=\"87\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545381797
Jul 17 03:10:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:49.526Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_scanCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", namespace=\"SYSTEM\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"e5edf26a618683345a7e32d6260d5b47\", table=\"LOG\", type=\"data\"}" timestamp=1689545387438
Jul 17 03:10:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:49.534Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"abrt-ccpp.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"service\"}" timestamp=1689545381057
Jul 17 03:10:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:49.541Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_193_1_metric_deleteBatchTime_95th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545387438
Jul 17 03:10:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:49.547Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_readRequestCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"default\", project=\"OLAP\", project_id=\"3\", region=\"668ca3489d8c6fec92a10db8b81f6e41\", table=\"index_partfile_default\", type=\"index\"}" timestamp=1689545387438
Jul 17 03:10:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:49.556Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_CheckAndDelete_90th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Server\"}" timestamp=1689545387438
Jul 17 03:10:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:49.587Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_queue_length\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethec314a7d\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545381797
Jul 17 03:10:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:49.607Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db167\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545386625
Jul 17 03:10:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:49.611Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-fd984c25\\\\x2d4c10\\\\x2d40eb\\\\x2da03c\\\\x2d2c7c403cf652-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2d2tf7t.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689545381057
Jul 17 03:10:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:49.627Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2dpartlabel-TSG\\\\x2dOS\\\\x2dUPDATE.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689545381057
Jul 17 03:10:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:49.643Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_down_changes_total\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth83674fd8\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545381797
Jul 17 03:10:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:49.814Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"twa-tsgx10\", asset_id=\"109\", condition=\"unknown\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-kube-state-metrics-6b768c7bf8-n4ck4\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"7be94d73-b8c6-4b83-adb9-10dac35bf41e\"}" timestamp=1689545379668
Jul 17 03:10:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:49.827Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-kube-state-metrics-6b768c7bf8-n4ck4\", project=\"TSG-OS\", project_id=\"7\", reason=\"Evicted\", service=\"prometheus-kube-state-metrics\", uid=\"7be94d73-b8c6-4b83-adb9-10dac35bf41e\"}" timestamp=1689545379668
Jul 17 03:10:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:49.868Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_reads_total\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"kube-state-metrics\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545363920
Jul 17 03:10:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:49.881Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_swap_limit_bytes\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"prometheus-alertmanager-configmap-reload\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545379050
Jul 17 03:10:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:49.922Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_io_time_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/var/lib/kubelet/pods/9d049c99-7658-4134-a1b4-cb83e78cc0a3/volumes/kubernetes.io~projected/kube-api-access-q4pnj\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545385566
Jul 17 03:10:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:49.925Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_scaling_frequency_hertz\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545386709
Jul 17 03:10:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:50.074Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_avail_bytes\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda1\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-olap003\", endpoint_id=\"235\", fstype=\"vfat\", module=\"OLAP-Node-Exporter\", module_id=\"51\", mountpoint=\"/boot/efi\", nz_agent_id=\"3\", olap_node_exporter=\"yarn_resourcemanager,zookeeper,nacos,kafka,spark,druid_data,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545389713
Jul 17 03:10:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:50.096Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db158\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545386625
Jul 17 03:10:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:50.124Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_type\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"chart-values-tsg-traffic-engine-numa-1\", service=\"prometheus-kube-state-metrics\", type=\"Opaque\"}" timestamp=1689545379668
Jul 17 03:10:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:50.141Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_errors_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", interface=\"ens6f1np1\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545386306
Jul 17 03:10:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:50.158Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"initrd-udevadm-cleanup-db.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"oneshot\", type=\"service\"}" timestamp=1689545381057
Jul 17 03:10:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:50.181Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"twa-tsgx08\", asset_id=\"105\", container=\"prometheus-pushgateway\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"running\"}" timestamp=1689545379856
Jul 17 03:10:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:50.282Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_colls_total\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethbd8ba9ba\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545381797
Jul 17 03:10:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:50.341Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"twa-tsgx10\", asset_id=\"109\", cpu=\"73\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545381797
Jul 17 03:10:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:50.363Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-kernel-debug-tracing.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689545381057
Jul 17 03:10:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:50.503Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db38\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545386625
Jul 17 03:10:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:50.588Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", cpu=\"29\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-olap003\", endpoint_id=\"235\", mode=\"nice\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"yarn_resourcemanager,zookeeper,nacos,kafka,spark,druid_data,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545389713
Jul 17 03:10:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:50.604Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", cpu=\"0\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-olap003\", endpoint_id=\"235\", mode=\"irq\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"yarn_resourcemanager,zookeeper,nacos,kafka,spark,druid_data,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545389713
Jul 17 03:10:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:50.646Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_annotations\", asset=\"twa-tsgx08\", asset_id=\"105\", configmap=\"kube-root-ca.crt\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545381568
Jul 17 03:10:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:50.691Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas_ready\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"coredns\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545381568
Jul 17 03:10:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:50.798Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_errors_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", interface=\"ens7f1np1\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545386306
Jul 17 03:10:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:50.983Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_io_time_weighted_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/data/var/lib/kubelet/pods/2cfe23ce-7e71-44dc-8e19-b936dd7c7f8c/volumes/kubernetes.io~projected/kube-api-access-msvmx\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545385566
Jul 17 03:10:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:51.040Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_load_average_10s\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"prometheus-pushgateway\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545370311
Jul 17 03:10:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:51.070Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_errors_total\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", err=\"LOADING\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545386625
Jul 17 03:10:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:51.186Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545386709
Jul 17 03:10:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:51.387Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_avail_bytes\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"shm\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/1b1a1d1c33e43c91f11f758ecd8b18e6e2f808ab06c667e1419586c784377212/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545386709
Jul 17 03:10:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:51.464Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_write_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"overlay_0-400\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545385566
Jul 17 03:10:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:52.437Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_created\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-0-firewall-7c999f565f\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545384478
Jul 17 03:10:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:52.516Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", container=\"dashboard-metrics-scraper\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Total\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545381261
Jul 17 03:10:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:52.523Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_labels\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"local-path-provisioner\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545384478
Jul 17 03:10:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:52.542Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", cpu=\"25\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545389337
Jul 17 03:10:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:52.565Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Unknown\", pod=\"helm-install-prometheus-snvfp\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"7e2a510b-5127-4fca-bfa1-de042913594f\"}" timestamp=1689545384478
Jul 17 03:10:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:52.616Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", cpu=\"41\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545389337
Jul 17 03:10:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:52.635Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-pod7af362d7_f44d_40a1_9857_36884a064be9.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"slice\"}" timestamp=1689545385496
Jul 17 03:10:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:52.662Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-virtual-misc-rfkill.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689545385496
Jul 17 03:10:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:52.904Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-801c91717cc1a419c3d0cf3fc8e577c5cbbf570d9c24a0cf8b8678f7d876782f.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"scope\"}" timestamp=1689545385496
Jul 17 03:10:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:53.071Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_memory_HugePages_Total\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545388328
Jul 17 03:10:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:53.108Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-4fdaf7e81625e64f32d7318c4ddcc46ab9bfb56ce2b058ed6fd3703b072dad92-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689545385496
Jul 17 03:10:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:53.147Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-c385b4a5a900a117708c9f54577244bcf20366eb87264f699990f6363627e73e-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689545385496
Jul 17 03:10:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:53.176Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", state=\"iowaiting\"}" timestamp=1689545374375
Jul 17 03:10:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:53.194Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_oom_events_total\", asset=\"msh-tsgx08\", asset_id=\"65\", container=\"dign-server-dns\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545371126
Jul 17 03:10:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:53.204Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_annotations\", asset=\"pcap-tsgx11\", asset_id=\"115\", configmap=\"kube-root-ca.crt\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545384478
Jul 17 03:10:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:53.209Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"msh-tsgx08\", asset_id=\"65\", condition=\"false\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-delete-tsg-traffic-engine-numa-0-gr2t2\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"9168d67a-9478-4ceb-b26f-40c0815c5d19\"}" timestamp=1689545388950
Jul 17 03:10:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:53.229Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", pod=\"dashboard-metrics-scraper-5b94897dc9-7xj9b\", project=\"TSG-OS\", project_id=\"7\", reason=\"NodeAffinity\", service=\"prometheus-kube-state-metrics\", uid=\"df708fd2-fb21-4af8-958e-4ba1dc3e2b4a\"}" timestamp=1689545388950
Jul 17 03:10:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:53.250Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx08\", asset_id=\"65\", cpu=\"0\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545388328
Jul 17 03:10:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:53.267Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"msh-tsgx08\", asset_id=\"65\", cpu=\"53\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545388328
Jul 17 03:10:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:53.291Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-2e7d9325bbdf9dcc63ffd58ef7384c922458ab78fc063599ce70d628f0ca4202.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"scope\"}" timestamp=1689545385496
Jul 17 03:10:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:53.356Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_drop_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth94114e67\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545389337
Jul 17 03:10:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:53.497Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"plymouth-quit-wait.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689545391179
Jul 17 03:10:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:53.538Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx12\", asset_id=\"113\", cpu=\"42\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545383391
Jul 17 03:10:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:53.597Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx08\", asset_id=\"65\", cpu=\"20\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545388328
Jul 17 03:10:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:53.614Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", container=\"redis\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"stopped\"}" timestamp=1689545380629
Jul 17 03:10:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:53.631Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_drop_total\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"vethae353140\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545388328
Jul 17 03:10:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:53.652Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_annotations\", asset=\"pcap-tsgx11\", asset_id=\"115\", configmap=\"certstore-tsg-traffic-engine-numa-0\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545384478
Jul 17 03:10:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:53.693Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_up_changes_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth561c9aba\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545389337
Jul 17 03:10:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:53.714Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2dpartuuid-32ea49c8\\\\x2decad\\\\x2d49a7\\\\x2da782\\\\x2da55e46941923.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689545391179
Jul 17 03:10:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:53.815Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_info\", asset=\"msh-tsgx08\", asset_id=\"65\", configmap=\"prometheus-alertmanager\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545388950
Jul 17 03:10:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:53.919Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_info\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.tsg-traffic-engine-numa-0.v1\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545388950
Jul 17 03:10:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:54.218Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_waiting\", asset=\"msh-tsgx12\", asset_id=\"113\", container=\"configmap-reloader\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"configmap-reloader-6c55bbc49f-tmsl2\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"3af07046-1fef-40d4-a68b-5b51a863820e\"}" timestamp=1689545388663
Jul 17 03:10:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:54.308Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx12\", asset_id=\"113\", cpu=\"41\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545383391
Jul 17 03:10:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:54.331Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cpu=\"97\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545379409
Jul 17 03:10:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:54.380Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_memory_Cached_bytes\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545379409
Jul 17 03:10:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:54.383Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"vgauthd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689545391179
Jul 17 03:10:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:54.410Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_sockets\", asset=\"msh-tsgx12\", asset_id=\"113\", container=\"exporter-merger\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545366074
Jul 17 03:10:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:54.412Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_up_changes_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens4f0np0\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545379409
Jul 17 03:10:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:54.438Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-f196657be2c3b0bf2404995c0c422d3e0bb9cb4ad02712aca22b3f82cc0dc335.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"scope\"}" timestamp=1689545391179
Jul 17 03:10:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:54.447Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Unknown\", pod=\"helm-install-ipmi-exporter-g8rxd\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"1486c485-089b-4414-90ed-8ef7a7e417f1\"}" timestamp=1689545382058
Jul 17 03:10:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:54.474Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_packets_total\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens8f1np1\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545383391
Jul 17 03:10:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:54.539Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_swap_limit_bytes\", asset=\"pcap-tsgx07\", asset_id=\"85\", container=\"telegraf\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689545390608
Jul 17 03:10:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:54.541Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"irqbalance.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"service\"}" timestamp=1689545391179
Jul 17 03:10:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:54.564Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", container=\"prometheus-server\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"iowaiting\"}" timestamp=1689545390162
Jul 17 03:10:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:54.571Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_init_container_status_ready\", asset=\"pcap-tsgx07\", asset_id=\"85\", container=\"install-multus-binary\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"kube-multus-ds-thbtz\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"fe410bd7-7bfd-4526-8256-38727d23f01b\"}" timestamp=1689545382058
Jul 17 03:10:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:54.624Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db14\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545385184
Jul 17 03:10:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:54.629Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-netns-cni\\\\x2d6911b04b\\\\x2d3741\\\\x2dd98e\\\\x2df645\\\\x2d662e0738ba71.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689545380884
Jul 17 03:10:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:54.648Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-ens3f0np0.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689545380884
Jul 17 03:10:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:54.694Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dracut-pre-trigger.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"service\"}" timestamp=1689545391179
Jul 17 03:10:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:54.881Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cpu=\"96\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545379409
Jul 17 03:10:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:54.897Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_flags\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"br_dign_c\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545379409
Jul 17 03:10:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:54.989Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx12\", asset_id=\"113\", cpu=\"105\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545383391
Jul 17 03:10:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:55.014Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_flags\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"lo\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545383391
Jul 17 03:10:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:55.031Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"cadvisor\", failure_type=\"pgfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"container\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689545374249
Jul 17 03:10:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:55.141Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"NetworkManager.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689545380884
Jul 17 03:10:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:55.186Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-2ed77d82\\\\x2dbe58\\\\x2d47ef\\\\x2d9e9a\\\\x2d55c3cc7c3703-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2dt52b4.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689545391179
Jul 17 03:10:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:55.188Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link_mode\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"vethcd3875c5\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545383391
Jul 17 03:10:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:55.204Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_protocol_type\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"vethfaa6969e\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545383391
Jul 17 03:10:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:55.207Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-sda.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689545391179
Jul 17 03:10:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:55.279Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-a902c527\\\\x2d6813\\\\x2d463a\\\\x2d85c3\\\\x2dee3b175b9cc9-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2dxkg6l.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689545380884
Jul 17 03:10:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:55.311Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db73\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545385184
Jul 17 03:10:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:55.431Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-a7cfa15dbba05667dbee87d84df2a784e59d9c20bb68873b5911c09871e25d79-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689545380884
Jul 17 03:10:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:56.356Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_system_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", container=\"local-path-provisioner\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545386501
Jul 17 03:10:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:56.372Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2did-scsi\\\\x2dSDELL_PERC_H345_Adp_0065e1fae1717c2c2c00578284f0a7ce\\\\x2dpart3.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689545383372
Jul 17 03:10:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:56.406Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"30\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545385883
Jul 17 03:10:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:56.436Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_constraint_errors_transmitted_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"mlx5_7\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545385883
Jul 17 03:10:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:56.477Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_packets_transmitted_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"mlx5_7\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545385883
Jul 17 03:10:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:56.630Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"dign-client-qkc6p\", project=\"TSG-OS\", project_id=\"7\", reason=\"Shutdown\", service=\"prometheus-kube-state-metrics\", uid=\"0acf4d0b-d722-41b5-aee0-e6bd3bc175ed\"}" timestamp=1689545393389
Jul 17 03:10:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:56.644Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_annotations\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.traefik-crd.v1\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545393389
Jul 17 03:10:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:56.875Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_spec_replicas\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"kubernetes-dashboard\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545393389
Jul 17 03:10:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:57.686Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"tmpfs\", name=\"data-var-lib-kubelet-pods-d1430253\\\\x2d8c7e\\\\x2d449d\\\\x2d9f2a\\\\x2ddd552c402bae-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2d45rz5.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689545383372
Jul 17 03:10:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:57.702Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"tmpfs\", name=\"var-lib-kubelet-pods-06d9f977\\\\x2d06a7\\\\x2d4069\\\\x2d895f\\\\x2de4ebce64d667-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2dzp98h.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689545383372
Jul 17 03:10:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:57.753Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_memory_VmallocChunk_bytes\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545385883
Jul 17 03:10:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:57.775Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_netstat_Tcp_InErrs\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545385883
Jul 17 03:10:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:57.787Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_info\", asset=\"pcap-tsgx03\", asset_id=\"73\", board_id=\"MT_0000000359\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"mlx5_5\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", firmware_version=\"22.28.4000\", hca_type=\"MT4125\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545385883
Jul 17 03:10:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:57.800Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"17\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545385883
Jul 17 03:10:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:57.989Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545385883
Jul 17 03:10:58 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:58.011Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_os_version\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", id=\"rocky\", id_like=\"rhel centos fedora\", module=\"TSG-OS\", module_id=\"77\", name=\"TSG-OS\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545385883
Jul 17 03:10:58 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:58.057Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_gc_cpu_fraction\", asset=\"msh-dellr740-dt003\", asset_id=\"9\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-dt003\", endpoint_id=\"229\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"hbase_master,clickhouse_data,kafka,gohangout,zookeeper,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545397407
Jul 17 03:10:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:59.733Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"4\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545389083
Jul 17 03:10:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:10:59.753Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_commands_duration_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cmd=\"del\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545385071
Jul 17 03:11:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:00.397Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_transmit_wait_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_8\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545389083
Jul 17 03:11:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:00.420Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"66\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545389083
Jul 17 03:11:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:00.457Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db106\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545385071
Jul 17 03:11:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:00.798Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_complete\", asset=\"msh-tsgx01\", asset_id=\"51\", condition=\"unknown\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", job_name=\"helm-install-prometheus\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545387425
Jul 17 03:11:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:00.851Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_commands_rejected_calls_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cmd=\"incrby\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545385071
Jul 17 03:11:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:00.867Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-d566c081f7ecaa1ce639fc83148816c9c9c1c90e3d18762e07c5b16cc00c5055.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"scope\"}" timestamp=1689545384879
Jul 17 03:11:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:01.158Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-infiniband-umad6.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689545384879
Jul 17 03:11:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:01.273Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"phy_tx_bits_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", dev=\"ens1f0np0\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"mrzcpd-exporter\"}" timestamp=1689545390377
Jul 17 03:11:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:02.828Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_max_usage_bytes\", asset=\"msh-tsgx02\", asset_id=\"63\", container=\"lb-tcp-443\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545383434
Jul 17 03:11:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:02.875Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_file_descriptors\", asset=\"msh-tsgx02\", asset_id=\"63\", container=\"telegraf\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689545383515
Jul 17 03:11:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:03.070Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_packets_total\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", interface=\"eno35np2\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545387652
Jul 17 03:11:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:03.095Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_usage_bytes\", asset=\"msh-tsgx02\", asset_id=\"63\", container=\"traefik\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545385939
Jul 17 03:11:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:03.215Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"tmpfs\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/var/lib/kubelet/pods/b03a57be-3cd5-4241-ac95-41b20a48d7ef/volumes/kubernetes.io~projected/kube-api-access-t7h92\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545398318
Jul 17 03:11:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:03.378Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"net_conntrack_dialer_conn_failed_total\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", dialer_name=\"139\", instance=\"10.10.20.159:10090\", reason=\"resolution\", type=\"global\"}" timestamp=1689545402989
Jul 17 03:11:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:03.512Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"net_conntrack_dialer_conn_attempted_total\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", dialer_name=\"203\", instance=\"10.10.20.159:10090\", type=\"global\"}" timestamp=1689545402989
Jul 17 03:11:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:03.605Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_device_id\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"bond0\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-dt001\", endpoint_id=\"225\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545402988
Jul 17 03:11:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:04.019Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_annotations\", asset=\"msh-tsgx02\", asset_id=\"63\", configmap=\"kube-root-ca.crt\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"consul\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545391412
Jul 17 03:11:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:04.184Z caller=dedupe.go:112 component=remote level=info remote_name=52db6f url=http://10.10.10.159:10090/nz-agent/prometheus/proxy/api/v1/write msg="Dropped sample for series that was not explicitly dropped via relabelling" ref=4436650
Jul 17 03:11:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:04.251Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"net_conntrack_dialer_conn_failed_total\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", dialer_name=\"395\", instance=\"10.10.20.159:10090\", reason=\"refused\", type=\"global\"}" timestamp=1689545402989
Jul 17 03:11:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:04.272Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"net_conntrack_dialer_conn_failed_total\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", dialer_name=\"401\", instance=\"10.10.20.159:10090\", reason=\"resolution\", type=\"global\"}" timestamp=1689545402989
Jul 17 03:11:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:04.358Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"prometheus_sd_discovered_targets\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", config=\"351\", datacenter=\"MSH\", instance=\"10.10.20.159:10090\", name=\"scrape\", type=\"global\"}" timestamp=1689545402989
Jul 17 03:11:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:04.847Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"prometheus_target_sync_length_seconds\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", instance=\"10.10.20.159:10090\", quantile=\"0.9\", scrape_job=\"229\", type=\"global\"}" timestamp=1689545402989
Jul 17 03:11:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:06.590Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:e0-0000:e0:03.1-0000:e2:00.1-infiniband_mad-umad11.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689545393582
Jul 17 03:11:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:06.614Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-virtual-net-veth1c375d86.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689545393582
Jul 17 03:11:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:06.838Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-cbd5f9c7f8c3e9174972a7661aa4472734b2a71f3714f3bdb2c867f4d10e75d7.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"scope\"}" timestamp=1689545393582
Jul 17 03:11:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:06.924Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_cpu_seconds_total\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"gssproxy.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689545393582
Jul 17 03:11:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:06.944Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_rss\", asset=\"twa-tsgx01\", asset_id=\"91\", container=\"prometheus-pushgateway\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545391963
Jul 17 03:11:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:06.982Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-c9cd2d343951639ada75f6442c40ab92c504e7e1a427ac84621b382c47742975.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"scope\"}" timestamp=1689545393582
Jul 17 03:11:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:07.028Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link_mode\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens4f0np0\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545403910
Jul 17 03:11:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:07.061Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_max_fds\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-udevd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689545393582
Jul 17 03:11:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:07.084Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-sda1.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689545393582
Jul 17 03:11:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:07.293Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_limit_bytes\", asset=\"twa-tsgx01\", asset_id=\"91\", container=\"metrics-server\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545402966
Jul 17 03:11:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:07.332Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_namespace_status_phase\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", phase=\"Terminating\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545401227
Jul 17 03:11:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:07.489Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_info\", asset=\"twa-tsgx01\", asset_id=\"91\", created_by_kind=\"ReplicaSet\", created_by_name=\"coredns-6d8dc697\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", host_ip=\"10.10.30.161\", host_network=\"false\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", node=\"localhost\", nz_agent_id=\"3\", pod=\"coredns-6d8dc697-dlsd9\", pod_ip=\"10.42.0.155\", priority_class=\"system-cluster-critical\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"4237b3dc-4d91-4979-b851-594354acfc12\"}" timestamp=1689545401227
Jul 17 03:11:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:07.506Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_multicast_packets_transmitted_total\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_11\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545403910
Jul 17 03:11:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:07.522Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"10\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545403910
Jul 17 03:11:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:07.551Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_address_assign_type\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens1f0np0\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545403910
Jul 17 03:11:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:07.821Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"17\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545403910
Jul 17 03:11:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:07.835Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_cpu_seconds_total\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"udisks2.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689545393582
Jul 17 03:11:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:07.857Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-binfmt.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"oneshot\", type=\"service\"}" timestamp=1689545393582
Jul 17 03:11:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:07.880Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Async\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689545384668
Jul 17 03:11:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:07.888Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_speed_bytes\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"eno34np1\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545403910
Jul 17 03:11:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:07.939Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cpu=\"27\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545403910
Jul 17 03:11:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:07.964Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.grpc.v1.cri-sandboxes-433f2d9f8d06db4f652b2dcc52e43ddde9df2f4692ca475334211f0301d15268-shm.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689545393582
Jul 17 03:11:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:07.983Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sshd-keygen@rsa.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"service\"}" timestamp=1689545393582
Jul 17 03:11:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:07.992Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"109\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545403910
Jul 17 03:11:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:08.219Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"phy_tx_bits_total\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", dev=\"virtio_dign_s\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"mrzcpd-exporter\"}" timestamp=1689545398833
Jul 17 03:11:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:08.229Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"twa-tsgx01\", asset_id=\"91\", condition=\"Available\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"tsg-traffic-engine-numa-1-firewall\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"unknown\"}" timestamp=1689545401227
Jul 17 03:11:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:08.384Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_observed_generation\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"systemd-exporter\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545401227
Jul 17 03:11:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:08.501Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_dormant\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth1db7f937\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545403910
Jul 17 03:11:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:08.515Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_id\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth76f0c56a\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545403910
Jul 17 03:11:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:10.530Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_putCount\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"knowledge_base_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"2195076ef2d2041d654651ace4a5a2ab\", table=\"index_partfile_knowledge_base_hos_bucket\", type=\"index\"}" timestamp=1689545408804
Jul 17 03:11:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:11.488Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_mobFileCacheCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Server\"}" timestamp=1689545410147
Jul 17 03:11:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:11.503Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_193_1_metric_majorCompactionInputFileCount_99th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545410147
Jul 17 03:11:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:11.551Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_session_record_hos_bucket_metric_appendTime_TimeRangeCount_0_1\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545410147
Jul 17 03:11:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:12.172Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_PreIndexUpdateFailureTime_90th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"PhoenixIndexer\"}" timestamp=1689545410147
Jul 17 03:11:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:12.184Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_storeFileSize\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"proxy_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"e94f3ce0c73fbb27c27dbf55d7fcafa9\", table=\"index_partfile_proxy_hos_bucket\", type=\"index\"}" timestamp=1689545410147
Jul 17 03:11:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:12.329Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_194_3_metric_tableWriteQueryPerSecond_1min_rate\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545410147
Jul 17 03:11:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:12.422Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_193_1_metric_deleteBatchTime_mean\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545410147
Jul 17 03:11:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:15.589Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_readonly\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"shm\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/5cc535b1876c0bcb3d5bfc70b104a2eda0ff079147786ef074810b0fedb27237/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545401230
Jul 17 03:11:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:15.625Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"18\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545401230
Jul 17 03:11:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:16.165Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Tcp_Offload_Pkt\", app_name=\"sapp4\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", vsysId=\"1\"}" timestamp=1689545400513
Jul 17 03:11:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:16.436Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_processes\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689545403005
Jul 17 03:11:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:16.457Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled_time\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-server-6ff784d758-sgqd4\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"0ed53af9-6e57-4bfe-bb64-b0aa4edce9c5\"}" timestamp=1689545408076
Jul 17 03:11:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:16.486Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_dormant\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth740df6ce\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545401230
Jul 17 03:11:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:19.615Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_limit_bytes\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/run\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545405027
Jul 17 03:11:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:19.638Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"92\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545417612
Jul 17 03:11:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:19.657Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_rdb_changes_since_last_save\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545411998
Jul 17 03:11:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:19.842Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_netstat_Tcp_InErrs\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545417612
Jul 17 03:11:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:19.866Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"114\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545417612
Jul 17 03:11:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:19.931Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-tmpfiles-setup-dev.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689545404356
Jul 17 03:11:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:20.238Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"2\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545417612
Jul 17 03:11:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:20.261Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_hwmon_temp_celsius\", asset=\"twa-tsgx06\", asset_id=\"101\", chip=\"pci0000:00_0000:00:18_3\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", sensor=\"temp3\", service=\"prometheus-node-exporter\"}" timestamp=1689545417612
Jul 17 03:11:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:20.309Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db42\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545411998
Jul 17 03:11:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:20.326Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"3\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545417612
Jul 17 03:11:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:20.482Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"41\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545417612
Jul 17 03:11:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:20.597Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"126\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545408263
Jul 17 03:11:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:20.612Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_queue_length\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"vetha31a3f11\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545408263
Jul 17 03:11:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:20.688Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_user_seconds_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", container=\"monit-stream\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545403996
Jul 17 03:11:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:20.957Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_vl15_dropped_total\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_2\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545417612
Jul 17 03:11:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:20.999Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_info\", asset=\"twa-tsgx06\", asset_id=\"101\", configmap=\"kube-root-ca.crt\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545413949
Jul 17 03:11:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:21.045Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", container=\"exporter-merger\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Sync\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545408348
Jul 17 03:11:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:21.074Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_annotations\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_endpoint=\"kubernetes-dashboard\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545413949
Jul 17 03:11:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:21.087Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_ready\", asset=\"twa-tsgx06\", asset_id=\"101\", container=\"helm\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-tsg-traffic-engine-numa-0-sfc4c\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"02e61820-d9ba-4a99-8e7c-ef3b109dc48f\"}" timestamp=1689545413949
Jul 17 03:11:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:21.095Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db189\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545411998
Jul 17 03:11:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:21.107Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-a583a07b462fb6b7ff36ff55d2f8f8e9f71a7b4eb049991cb9307970b920978b.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"scope\"}" timestamp=1689545404356
Jul 17 03:11:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:21.112Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"96\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545408263
Jul 17 03:11:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:21.129Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"tmpfs\", name=\"run-k3s-containerd-io.containerd.grpc.v1.cri-sandboxes-e143fc944a59d6130b45a4e58a23386fe81a0810c3710bfb7067d12f15fe21a2-shm.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689545410657
Jul 17 03:11:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:21.152Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_waiting\", asset=\"pcap-tsgx10\", asset_id=\"89\", container=\"kube-state-metrics\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-kube-state-metrics-6b768c7bf8-xsfwc\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"10741627-0ed0-416d-ac50-5a8214dc1db7\"}" timestamp=1689545413828
Jul 17 03:11:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:21.172Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_netstat_Icmp6_InErrors\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545408263
Jul 17 03:11:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:21.188Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_frame_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens2f1np1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545408263
Jul 17 03:11:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:21.245Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"111\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545417612
Jul 17 03:11:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:21.601Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-ab53e5a7\\\\x2df22e\\\\x2d498d\\\\x2db5af\\\\x2d535ae7359350-volume\\\\x2dsubpaths-config\\\\x2dvolume-telegraf-2.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689545410657
Jul 17 03:11:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:21.832Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_goroutines\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"OLAP-Node-Exporter-twa-dellr740-dt001\", endpoint_id=\"257\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545421405
Jul 17 03:11:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:22.360Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_files_free\", asset=\"msh-dellr740-olap004\", asset_id=\"17\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"tmpfs\", endpoint=\"NC-Spark-msh-dellr740-olap004\", endpoint_id=\"15\", fstype=\"tmpfs\", module=\"NC-Spark\", module_id=\"47\", mountpoint=\"/run/user/0\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545422253
Jul 17 03:11:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:22.590Z caller=dedupe.go:112 component=remote level=error remote_name=52db6f url=http://10.10.10.159:10090/nz-agent/prometheus/proxy/api/v1/write msg="non-recoverable error" count=1000 exemplarCount=0 err="server returned HTTP status 400 : out of order sample"
Jul 17 03:11:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:22.869Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"clickhouse_up\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"OLAP-Node-Exporter-twa-dellr740-dt001\", endpoint_id=\"257\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545421405
Jul 17 03:11:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:23.160Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"jvm_memory_bytes_init\", area=\"heap\", asset=\"msh-dellr740-dt003\", asset_id=\"9\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-Yarn-msh-dellr740-dt003-NodeManager\", endpoint_id=\"355\", module=\"DC-Yarn\", module_id=\"1\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545422405
Jul 17 03:11:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:23.288Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_frequency_min_hertz\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", cpu=\"4\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"OLAP-Node-Exporter-twa-dellr740-dt001\", endpoint_id=\"257\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545421405
Jul 17 03:11:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:23.523Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_owner\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", owner_is_controller=\"true\", owner_kind=\"Job\", owner_name=\"helm-install-tsg-traffic-engine-numa-0\", pod=\"helm-install-tsg-traffic-engine-numa-0-n2fbk\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"044ab283-1226-4087-a5d1-3cb614606766\"}" timestamp=1689545420068
Jul 17 03:11:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:23.538Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_labels\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.prometheus.v1\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545420068
Jul 17 03:11:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:23.553Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_size_bytes\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"shm\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/852b631a3baff6085e592e514448bfb1292ec41588c97b9cc1e9c05320126a83/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545414790
Jul 17 03:11:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:24.004Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx09\", asset_id=\"55\", cpu=\"85\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545414790
Jul 17 03:11:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:24.061Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_scrape_collector_duration_seconds\", asset=\"msh-tsgx09\", asset_id=\"55\", collector=\"filesystem\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545414790
Jul 17 03:11:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:24.197Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_limit_bytes\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda5\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545409512
Jul 17 03:11:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:24.215Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_observed_generation\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"systemd-exporter-7f549d9485\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545420068
Jul 17 03:11:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:24.219Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas_unavailable\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"prometheus-server\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545420068
Jul 17 03:11:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:24.237Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_device_id\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens4f1np1\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545414790
Jul 17 03:11:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:24.256Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_cpu_seconds_total\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-pode335f8b4_74cf_439e_9760_c54f88f85a5d.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"slice\"}" timestamp=1689545418416
Jul 17 03:11:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:24.366Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens6f0np0\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545414790
Jul 17 03:11:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:24.387Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:e0-0000:e0:03.1-0000:e2:00.1-infiniband_mad-umad11.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689545418416
Jul 17 03:11:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:24.439Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_state_started\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"metrics\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"cm-policy-local-cache-redis-master-69967b5879-8m6nj\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"ebdf9219-42ed-4a26-8500-14f2dacdb9fe\"}" timestamp=1689545411122
Jul 17 03:11:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:24.440Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"up\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Druid-msh-dellr740-olap003\", endpoint_id=\"383\", instance=\"192.168.20.223:9903\", job=\"383\", module=\"NC-Druid\", module_id=\"31\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545423363
Jul 17 03:11:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:24.451Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_ips\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", ip=\"10.42.0.160\", ip_family=\"4\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-server-6ff784d758-cs9wf\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"58676ba8-2676-43fd-b12b-038d471289b5\"}" timestamp=1689545420068
Jul 17 03:11:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:24.457Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Unknown\", pod=\"helm-install-prometheus-glllf\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"8262b103-7c5a-40c8-9681-3282084e7b13\"}" timestamp=1689545411122
Jul 17 03:11:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:24.475Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-d9a3f5f77488b540fc9cffd67fb15e9f552a6cc28c6f84d52ab986ec842d0d86.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"scope\"}" timestamp=1689545411102
Jul 17 03:11:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:24.476Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_drop_total\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"vethdd79d05f\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545414790
Jul 17 03:11:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:24.568Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_packets_transmitted_total\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_7\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545414790
Jul 17 03:11:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:24.590Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_drop_total\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens7f1np1\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545414790
Jul 17 03:11:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:24.793Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_files_free\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda5\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", fstype=\"ext4\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/var\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545420160
Jul 17 03:11:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:24.816Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth3cf0d389\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545420160
Jul 17 03:11:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:24.825Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_virtual_memory_max_bytes\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"smartd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689545418416
Jul 17 03:11:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:24.843Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-28c4c7954fe67a1a5a5668e4a62d2b4acee44e9b6b2f9f6923cc5368368e67be-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689545418416
Jul 17 03:11:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:24.849Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failcnt\", asset=\"msh-tsgx09\", asset_id=\"55\", container=\"lb-tcp-80\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545414657
Jul 17 03:11:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:24.871Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_ports\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_endpoint=\"kube-dns\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", port_name=\"dns\", port_number=\"53\", port_protocol=\"UDP\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545420068
Jul 17 03:11:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:24.988Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_info\", asset=\"msh-tsgx09\", asset_id=\"55\", configmap=\"local-path-config\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545420068
Jul 17 03:11:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:25.019Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_multicast_packets_received_total\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_0\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545414790
Jul 17 03:11:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:25.055Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_scrape_collector_success\", asset=\"msh-tsgx09\", asset_id=\"55\", collector=\"diskstats\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545414790
Jul 17 03:11:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:25.419Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"91\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545420160
Jul 17 03:11:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:25.507Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"shutdown.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"target\"}" timestamp=1689545411102
Jul 17 03:11:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:26.039Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-mrzcpd-exporter-pns2w\", project=\"TSG-OS\", project_id=\"7\", reason=\"NodeLost\", service=\"prometheus-kube-state-metrics\", uid=\"524fda03-478b-44e7-9ee9-f14d8b203a6f\"}" timestamp=1689545418744
Jul 17 03:11:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:26.421Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_oom_events_total\", asset=\"twa-tsgx09\", asset_id=\"107\", container=\"exporter\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545422903
Jul 17 03:11:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:26.439Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_status_succeeded\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", job_name=\"helm-install-tsg-traffic-engine-numa-0\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545418744
Jul 17 03:11:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:26.548Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_compressed_total\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth1e7f8cb3\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545415810
Jul 17 03:11:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:26.567Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_fifo_total\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens4f1np1\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545415810
Jul 17 03:11:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:26.824Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"cadvisor\", failure_type=\"pgmajfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"hierarchy\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689545424126
Jul 17 03:11:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:27.029Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_up\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethb8a1e98e\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545415810
Jul 17 03:11:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:27.056Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"120\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545415810
Jul 17 03:11:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:27.074Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"92\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545415810
Jul 17 03:11:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:27.089Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_transmit_wait_total\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_6\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545415810
Jul 17 03:11:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:27.175Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_sockets\", asset=\"twa-tsgx09\", asset_id=\"107\", container=\"metrics-server\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545411977
Jul 17 03:11:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:27.209Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_threads\", asset=\"twa-tsgx09\", asset_id=\"107\", container=\"ipmi-exporter\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545419216
Jul 17 03:11:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:27.307Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"5\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545415810
Jul 17 03:11:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:27.340Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link_mode\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethb8a1e98e\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545415810
Jul 17 03:11:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:27.402Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_last_seen\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"dashboard-metrics-scraper\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545424865
Jul 17 03:11:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:27.417Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_queue_length\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"br_dign_c\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545422882
Jul 17 03:11:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:27.433Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"rescue.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"target\"}" timestamp=1689545414114
Jul 17 03:11:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:27.592Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_errs_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth875ea1eb\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545422882
Jul 17 03:11:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:27.596Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_threads_max\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"prometheus-pushgateway\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545418694
Jul 17 03:11:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:27.607Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_drop_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"virtio_dign_c\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545422882
Jul 17 03:11:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:27.696Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_multicast_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens3f0np0\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545422882
Jul 17 03:11:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:27.733Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_fifo_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens8f1np1\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545422882
Jul 17 03:11:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:27.825Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_reads_total\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"lb-tcp-80\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545407475
Jul 17 03:11:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:27.838Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_id\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth33f6ff75\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545422882
Jul 17 03:11:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:27.876Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_virtual_memory_bytes\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-journald.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689545414114
Jul 17 03:11:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:28.046Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_swap\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"exporter-merger\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545412408
Jul 17 03:11:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:28.101Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_spec_type\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"shaping-announce-port-tsg-traffic-engine-numa-0\", type=\"NodePort\", uid=\"796b9ec8-8607-4de8-b81e-4476eafa0e4c\"}" timestamp=1689545415682
Jul 17 03:11:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:28.151Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_address_assign_type\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"eno34np1\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545422882
Jul 17 03:11:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:28.167Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db236\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545419446
Jul 17 03:11:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:28.397Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"firewall\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Total\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689545423356
Jul 17 03:11:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:28.477Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-prometheus-fbsht\", project=\"TSG-OS\", project_id=\"7\", reason=\"NodeLost\", service=\"prometheus-kube-state-metrics\", uid=\"bb5f85db-6a6f-4d8a-9d37-be7ad4071f86\"}" timestamp=1689545415682
Jul 17 03:11:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:28.478Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_status_failed\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", job_name=\"helm-install-configmap-reloader\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545415682
Jul 17 03:11:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:28.522Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"msh-tsgx10\", asset_id=\"69\", condition=\"unknown\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", pod=\"dashboard-metrics-scraper-5b94897dc9-8srnj\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"c0c07089-8d67-4a93-9c72-855e72ebd4e3\"}" timestamp=1689545415682
Jul 17 03:11:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:28.527Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_frame_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth6714ba50\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545422882
Jul 17 03:11:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:28.776Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"msh-tsgx10\", asset_id=\"69\", condition=\"false\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-kube-state-metrics-6b768c7bf8-c9qwx\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"8cf57b14-cf54-45b0-aa29-1e25506f5e09\"}" timestamp=1689545415682
Jul 17 03:11:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:28.853Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_info\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.ipmi-exporter.v1\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545415682
Jul 17 03:11:29 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:29.037Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"redis\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", failure_type=\"pgfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"hierarchy\"}" timestamp=1689545412740
Jul 17 03:11:29 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:29.093Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"52\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545422882
Jul 17 03:11:29 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:29.118Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_drop_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth875ea1eb\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545422882
Jul 17 03:11:29 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:29.306Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"msh-tsgx10\", asset_id=\"69\", condition=\"Available\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"tsg-traffic-engine-numa-0-firewall\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"true\"}" timestamp=1689545415682
Jul 17 03:11:29 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:29.332Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_labels\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.mrzcpd-exporter.v1\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545415682
Jul 17 03:11:29 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:29.357Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_info\", asset=\"msh-tsgx10\", asset_id=\"69\", board_id=\"MT_0000000359\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_2\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", firmware_version=\"22.28.4000\", hca_type=\"MT4125\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545422882
Jul 17 03:11:29 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:29.458Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens6f0np0\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545422882
Jul 17 03:11:29 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:29.487Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_queue_length\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth30e28f20\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545422882
Jul 17 03:11:29 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:29.535Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_cpu_period\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"telegraf\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689545424841
Jul 17 03:11:29 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:29.636Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth6f518146\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545422882
Jul 17 03:11:29 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:29.978Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kafka_server_socket_server_metrics_request_rate\", asset=\"twa-dellr740-dt002\", asset_id=\"39\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-Kafka-twa-dellr740-dt002\", endpoint_id=\"107\", listener=\"SSL\", module=\"DC-Kafka\", module_id=\"5\", networkProcessor=\"6\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545429470
Jul 17 03:11:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:31.016Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"19\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545420939
Jul 17 03:11:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:31.034Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_drop_total\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"lo\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545420939
Jul 17 03:11:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:31.085Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"73\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545420939
Jul 17 03:11:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:31.094Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"97\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545420939
Jul 17 03:11:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:31.118Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_errs_total\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth283b535f\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545420939
Jul 17 03:11:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:31.121Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_transmit_wait_total\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_8\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545420939
Jul 17 03:11:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:31.238Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_carrier_total\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"eno2\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545420939
Jul 17 03:11:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:31.422Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_state_started\", asset=\"msh-tsgx05\", asset_id=\"67\", container=\"helm\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-prometheus-vv7rh\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"78da91fa-de71-46c4-a10d-8e983d0c12de\"}" timestamp=1689545424068
Jul 17 03:11:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:31.440Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-4f98f519dd409b9d783c2c8b3549ff189ddfd5da18a73f8579ac015126fe00ae.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"scope\"}" timestamp=1689545421015
Jul 17 03:11:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:31.513Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_metadata_generation\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"ipmi-exporter-6f8464dd94\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545430436
Jul 17 03:11:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:31.531Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_address_assign_type\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethfd281763\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545419093
Jul 17 03:11:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:31.598Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_observed_generation\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-1-firewall-6d869d4b4c\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545424068
Jul 17 03:11:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:31.631Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-virtual-net-vethdc7cf888.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689545421015
Jul 17 03:11:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:31.717Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"49\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545420791
Jul 17 03:11:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:31.761Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link_mode\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"vethe4b4946d\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545420791
Jul 17 03:11:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:31.799Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda2\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", fstype=\"ext4\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/mnt/onie-boot\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545420791
Jul 17 03:11:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:32.003Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"69\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545419093
Jul 17 03:11:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:32.089Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_complete\", asset=\"twa-tsgx03\", asset_id=\"95\", condition=\"unknown\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", job_name=\"helm-install-prometheus\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545430436
Jul 17 03:11:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:32.158Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_observed_generation\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"dashboard-metrics-scraper-5b94897dc9\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545430436
Jul 17 03:11:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:32.197Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_status_active\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", job_name=\"helm-install-traefik\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545424068
Jul 17 03:11:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:32.217Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_fifo_total\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"virtio_dign_s\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545419093
Jul 17 03:11:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:32.264Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_id\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth2cff15d8\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545420939
Jul 17 03:11:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:32.349Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545419093
Jul 17 03:11:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:32.364Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"57\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545419093
Jul 17 03:11:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:32.504Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db31\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545426438
Jul 17 03:11:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:32.742Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_restarts_total\", asset=\"twa-tsgx05\", asset_id=\"99\", container=\"dign-server-dns\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"dign-server-nrgkb\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"48017225-2801-4344-bec5-b6556a71e4cf\"}" timestamp=1689545419894
Jul 17 03:11:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:32.792Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"promhttp_metric_handler_errors_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cause=\"encoding\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545420791
Jul 17 03:11:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:32.793Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_speed_bytes\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"cnibr0\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545420605
Jul 17 03:11:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:33.178Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_changes_total\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth50f7b1ca\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545420605
Jul 17 03:11:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:33.321Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_fifo_total\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth6de2e73d\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545420605
Jul 17 03:11:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:33.365Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.grpc.v1.cri-sandboxes-3fa6bbcd92c9fd5ab2e7cec52811c26506212df7c17327e3bbbfef5c670783e0-shm.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689545427691
Jul 17 03:11:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:33.403Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-netns-cni\\\\x2d98ff062d\\\\x2d8956\\\\x2d0c4f\\\\x2ddbf5\\\\x2df85cf156c77a.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689545422644
Jul 17 03:11:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:33.725Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545420605
Jul 17 03:11:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:33.797Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link_mode\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens4f1np1\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545420939
Jul 17 03:11:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:34.221Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_init_container_status_ready\", asset=\"twa-tsgx05\", asset_id=\"99\", container=\"firewall-init\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", pod=\"tsg-traffic-engine-numa-0-firewall-7698d7c499-8zkwn\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"9982afd0-5736-4dce-b764-2f7b0c30831a\"}" timestamp=1689545419894
Jul 17 03:11:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:34.236Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"twa-tsgx03\", asset_id=\"95\", container=\"telegraf-security\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"cadvisor\", failure_type=\"pgfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"container\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689545414465
Jul 17 03:11:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:34.256Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_spec_parallelism\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", job_name=\"helm-install-traefik\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545430436
Jul 17 03:11:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:34.292Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_last_gc_time_seconds\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545420605
Jul 17 03:11:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:34.314Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_commands_rejected_calls_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cmd=\"config\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545426438
Jul 17 03:11:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:34.329Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_ports\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_endpoint=\"kube-dns\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", port_name=\"dns-tcp\", port_number=\"53\", port_protocol=\"TCP\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545419894
Jul 17 03:11:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:34.343Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"110\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545419093
Jul 17 03:11:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:34.360Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_terminated_reason\", asset=\"twa-tsgx05\", asset_id=\"99\", container=\"helm\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-ipmi-exporter-qg4pr\", project=\"TSG-OS\", project_id=\"7\", reason=\"Completed\", service=\"prometheus-kube-state-metrics\", uid=\"6d488923-85e6-4a2b-a06b-4a00d69292b7\"}" timestamp=1689545419894
Jul 17 03:11:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:34.372Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"tmpfs\", name=\"run-k3s-containerd-io.containerd.grpc.v1.cri-sandboxes-e7ef0f6f76aa9c08b93f4f2e538436f3e9ab662e80c1533b8e2a80f554e3a0da-shm.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689545422644
Jul 17 03:11:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:34.373Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_annotations\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", job_name=\"helm-install-traefik-crd\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545430436
Jul 17 03:11:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:34.426Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"rpc_pipefs\", name=\"var-lib-nfs-rpc_pipefs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689545427691
Jul 17 03:11:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:34.429Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"microcode.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"service\"}" timestamp=1689545426907
Jul 17 03:11:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:34.464Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db26\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545429260
Jul 17 03:11:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:34.502Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_info\", asset=\"twa-tsgx03\", asset_id=\"95\", container=\"monit-stream\", container_id=\"containerd://446c013caa834ae734d87fd2758c12841e12b4690f9b337d21b0ac72876b4ae5\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", image=\"registry.gdnt-cloud.website/mrzcpd-exporter:1.0.1\", image_id=\"sha256:b1a7adfb61be24490a1321c37405d86204b78b1ee251ed959d48dc357310fb1c\", image_spec=\"registry.gdnt-cloud.website/mrzcpd-exporter:1.0.1\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"mrzcpd-exporter-5cf7f967cc-q54bz\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"a39ec036-7283-4e71-8c2a-9153e31b8d35\"}" timestamp=1689545430436
Jul 17 03:11:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:34.524Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_init_container_status_terminated\", asset=\"twa-tsgx03\", asset_id=\"95\", container=\"install-multus-binary\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"kube-multus-ds-8krwn\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"da4fef9f-6e38-4557-ac6b-3ab5eb04a34f\"}" timestamp=1689545430436
Jul 17 03:11:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:34.622Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_bytes_total\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"br_dign_s\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545420605
Jul 17 03:11:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:34.662Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db245\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545429260
Jul 17 03:11:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:34.682Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Tcp_Offload_Pkt\", app_name=\"sapp4\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", vsysId=\"1\"}" timestamp=1689545419140
Jul 17 03:11:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:34.845Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"73\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545420791
Jul 17 03:11:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:34.857Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_link_downed_total\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545420791
Jul 17 03:11:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:34.882Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_disk_io_time_seconds_total\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"sda\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545420791
Jul 17 03:11:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:34.907Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_device_id\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens1f0np0\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545420791
Jul 17 03:11:35 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:35.799Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx05\", asset_id=\"99\", container=\"dashboard-metrics-scraper\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Total\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545413559
Jul 17 03:11:35 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:35.819Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_annotations\", asset=\"twa-tsgx05\", asset_id=\"99\", configmap=\"kube-root-ca.crt\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-public\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545419894
Jul 17 03:11:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:40.125Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-dellr740-dt003\", asset_id=\"41\", cpu=\"22\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"OLAP-Node-Exporter-twa-dellr740-dt003\", endpoint_id=\"261\", mode=\"softirq\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"hbase_master,clickhouse_data,kafka,gohangout,zookeeper,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545438473
Jul 17 03:11:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:40.591Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_running\", asset=\"twa-tsgx04\", asset_id=\"97\", container=\"dign-server-dns\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"dign-server-7zllz\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"a76c8dce-9010-4080-b63c-3802527a42aa\"}" timestamp=1689545440034
Jul 17 03:11:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:40.857Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_SyncTime_median\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"WAL\"}" timestamp=1689545439756
Jul 17 03:11:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:41.156Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_spec_replicas\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-0-firewall-7d47f9b4d5\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545440034
Jul 17 03:11:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:41.295Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_appendCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"firewall_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"57210503f594ede560ea641f752e077d\", table=\"index_filename_firewall_hos_bucket\", type=\"index\"}" timestamp=1689545439756
Jul 17 03:11:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:41.352Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_numMasterWALs\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-Hmaster\", endpoint_id=\"339\", module=\"DC-HBase\", module_id=\"3\", name=\"Master\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Procedure\"}" timestamp=1689545440907
Jul 17 03:11:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:41.361Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"home.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689545435764
Jul 17 03:11:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:41.438Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"paths.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"target\"}" timestamp=1689545435764
Jul 17 03:11:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:41.712Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_memory_MemAvailable_bytes\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545431647
Jul 17 03:11:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:42.173Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_maxStoreFileAge\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"session_record_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"ac53627af7eafd3195f127d0f286f4b5\", table=\"index_time_session_record_hos_bucket\", type=\"index\"}" timestamp=1689545439756
Jul 17 03:11:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:42.201Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_SplitTime_99th_percentile\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Server\"}" timestamp=1689545439756
Jul 17 03:11:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:42.247Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"8\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545431647
Jul 17 03:11:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:42.410Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"iscsid.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"service\"}" timestamp=1689545435764
Jul 17 03:11:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:42.520Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_spec_volumes_persistentvolumeclaims_readonly\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", persistentvolumeclaim=\"prometheus-server\", pod=\"prometheus-server-6ff784d758-bfmc9\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"497bfad5-323f-4476-8405-f97cffdaa52c\", volume=\"storage-volume\"}" timestamp=1689545440034
Jul 17 03:11:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:42.537Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"initrd-switch-root.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"service\"}" timestamp=1689545435764
Jul 17 03:11:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:42.549Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_putCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"642211bde8d5f0835bfb2d2da5136308\", table=\"default\", type=\"data\"}" timestamp=1689545439756
Jul 17 03:11:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:42.562Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_reservation_limit_bytes\", asset=\"pcap-tsgx01\", asset_id=\"71\", container=\"telegraf-security\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689545440951
Jul 17 03:11:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:42.689Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_writes_bytes_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", container=\"telegraf\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689545423226
Jul 17 03:11:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:42.758Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"120\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545431647
Jul 17 03:11:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:42.776Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_annotations\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", job_name=\"helm-install-tsg-traffic-engine-numa-0\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545435154
Jul 17 03:11:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:42.834Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"abrt-xorg.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"service\"}" timestamp=1689545440671
Jul 17 03:11:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:42.913Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", cpu=\"116\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545436242
Jul 17 03:11:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:43.032Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", cpu=\"75\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545436242
Jul 17 03:11:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:43.276Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_fifo_total\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethefffe982\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545431647
Jul 17 03:11:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:43.574Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", cpu=\"32\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545436242
Jul 17 03:11:45 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:45.343Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db193\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545434838
Jul 17 03:11:45 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:45.869Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_usage_seconds_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", container=\"metrics-server\", cpu=\"total\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545427840
Jul 17 03:11:46 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:46.062Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-0cfcc16a\\\\x2dd96c\\\\x2d4497\\\\x2d8810\\\\x2d811308fc4c9e-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2dd54lv.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689545440671
Jul 17 03:11:48 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:48.522Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"clickhouse_cpu_frequency_m_hz_20\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Clickhouse-msh-dellr740-dt001\", endpoint_id=\"79\", module=\"NC-Clickhouse\", module_id=\"29\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545447155
Jul 17 03:11:48 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:48.944Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_maxStoreFileAge\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"81707482c620eb510b89508d73a4c9f7\", table=\"knowledge_base_hos_bucket\", type=\"data\"}" timestamp=1689545447447
Jul 17 03:11:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:49.208Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"java_lang_MemoryPool_PeakUsage_max\", asset=\"pcap-dellr740-dt003\", asset_id=\"35\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt003-Hmaster\", endpoint_id=\"387\", module=\"DC-HBase\", module_id=\"3\", name=\"Code Cache\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545446370
Jul 17 03:11:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:49.563Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_exceptions_FailedSanityCheckException\", asset=\"pcap-dellr740-dt003\", asset_id=\"35\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt003-Hmaster\", endpoint_id=\"387\", module=\"DC-HBase\", module_id=\"3\", name=\"Master\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"IPC\"}" timestamp=1689545446370
Jul 17 03:11:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:49.731Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_sockets\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"telegraf-security\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689545427356
Jul 17 03:11:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:49.787Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_FsReadTime_mean\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"IO\"}" timestamp=1689545447447
Jul 17 03:11:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:49.831Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_compactionsFailedCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"proxy_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"211911888371e8f4b47031fc3c87006b\", table=\"index_filename_proxy_hos_bucket\", type=\"index\"}" timestamp=1689545447447
Jul 17 03:11:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:49.873Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"tmpfs\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/data/var/lib/kubelet/pods/6f7082c9-24a1-4f4f-af11-ba474eb9a99b/volumes/kubernetes.io~projected/kube-api-access-m8hpv\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545446709
Jul 17 03:11:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:49.970Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", cpu=\"10\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-olap003\", endpoint_id=\"235\", mode=\"system\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"yarn_resourcemanager,zookeeper,nacos,kafka,spark,druid_data,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545449713
Jul 17 03:11:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:49.993Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_io_time_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/d19a49450e5c571bd50dc400211e0fcc9b8c589d40b6151bdb8d993be7c680ec/shm\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545445909
Jul 17 03:11:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:50.059Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_heap_released_bytes\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545447556
Jul 17 03:11:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:50.104Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"twa-tsgx08\", asset_id=\"105\", condition=\"true\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"dign-client-pv7jg\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"f9e681d8-8372-4551-8580-4a68b3fae8df\"}" timestamp=1689545441568
Jul 17 03:11:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:50.339Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_labels\", asset=\"twa-tsgx08\", asset_id=\"105\", configmap=\"cm-policy-local-cache-redis-configuration\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545441568
Jul 17 03:11:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:50.444Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-rdma-devices-mlx5_9:1-umad.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689545446554
Jul 17 03:11:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:50.645Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_files_free\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"shm\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/3ef3636b7f5d7d7024841cdb198e770c0990da29955e2d43716f9fd969032bfa/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545446709
Jul 17 03:11:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:50.833Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-6f7082c9\\\\x2d24a1\\\\x2d4f4f\\\\x2daf11\\\\x2dba474eb9a99b-volume\\\\x2dsubpaths-telegraf-telegraf\\\\x2dsecurity-0.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689545446554
Jul 17 03:11:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:51.096Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_annotations\", asset=\"twa-tsgx10\", asset_id=\"109\", configmap=\"kube-root-ca.crt\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"consul\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545439668
Jul 17 03:11:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:51.442Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"twa-tsgx10\", asset_id=\"109\", cpu=\"16\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545441797
Jul 17 03:11:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:51.535Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_incrementCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"rtp_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"b242966859aa9721f3ac6e8b17d51a4d\", table=\"index_filename_rtp_hos_bucket\", type=\"index\"}" timestamp=1689545447447
Jul 17 03:11:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:51.626Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_ips\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", ip=\"10.42.0.111\", ip_family=\"4\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"local-path-provisioner-957fdf8bc-5w5bc\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"28e75c4d-4cd3-41cb-8435-dd13365e6e7d\"}" timestamp=1689545439668
Jul 17 03:11:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:51.666Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_scrape_collector_duration_seconds\", asset=\"twa-tsgx10\", asset_id=\"109\", collector=\"schedstat\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545441797
Jul 17 03:11:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:51.692Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_excessive_buffer_overrun_errors_total\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_11\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545441797
Jul 17 03:11:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:51.745Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_commands_duration_seconds_total\", asset=\"twa-tsgx10\", asset_id=\"109\", cmd=\"ping\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545446625
Jul 17 03:11:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:51.775Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_scanCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"proxy_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"b805b36da9bcf501ffe4cb7944b5bda1\", table=\"index_time_proxy_hos_bucket\", type=\"index\"}" timestamp=1689545447447
Jul 17 03:11:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:51.812Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_2_metric_majorCompactionOutputFileCount_max\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545447447
Jul 17 03:11:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:51.845Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_writes_merged_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"overlay_0-123\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545445909
Jul 17 03:11:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:52.023Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_limit_bytes\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"dign-server-web\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545439050
Jul 17 03:11:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:52.608Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"selinux-autorelabel-mark.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"service\"}" timestamp=1689545441057
Jul 17 03:11:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:52.620Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-virtual-net-vethe7bf01ed.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689545441057
Jul 17 03:11:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:52.711Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_DDOS_Bypass\", app_name=\"sapp4\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", vsysId=\"1\"}" timestamp=1689545447223
Jul 17 03:11:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:52.756Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_compressed_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens2f0np0\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545446709
Jul 17 03:11:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:52.827Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_up\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethbec6d7d4\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545446709
Jul 17 03:11:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:52.988Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_physical_state_id\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"mlx5_1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545449337
Jul 17 03:11:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:53.184Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-ens7f0np0.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689545446554
Jul 17 03:11:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:53.276Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_memory_CommitLimit_bytes\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545446709
Jul 17 03:11:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:53.399Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db111\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545447556
Jul 17 03:11:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:53.689Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", cpu=\"21\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-olap003\", endpoint_id=\"235\", mode=\"iowait\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"yarn_resourcemanager,zookeeper,nacos,kafka,spark,druid_data,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545449713
Jul 17 03:11:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:53.737Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link_mode\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"lo\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545449337
Jul 17 03:11:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:53.769Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_type\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"kubernetes-dashboard-csrf\", service=\"prometheus-kube-state-metrics\", type=\"Opaque\"}" timestamp=1689545448950
Jul 17 03:11:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:53.782Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_compressed_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"eno36np3\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545449337
Jul 17 03:11:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:53.834Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_address_assign_type\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"bond0\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545446709
Jul 17 03:11:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:54.013Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_swap_limit_bytes\", asset=\"msh-tsgx12\", asset_id=\"113\", container=\"dign-server-dns\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545440589
Jul 17 03:11:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:54.031Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"msh-tsgx12\", asset_id=\"113\", condition=\"true\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"kube-multus-ds-87zqt\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"8b2e9710-4fd8-4c6f-8e5f-798d330e80a6\"}" timestamp=1689545448663
Jul 17 03:11:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:54.055Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"msh-tsgx08\", asset_id=\"65\", cpu=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545448328
Jul 17 03:11:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:54.075Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db144\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545445184
Jul 17 03:11:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:54.082Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:c0-0000:c0:01.1-0000:c1:00.1-infiniband-mlx5_9.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689545452135
Jul 17 03:11:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:54.125Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_fifo_total\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens3f0np0\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545443391
Jul 17 03:11:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:54.166Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_sockets\", asset=\"msh-tsgx12\", asset_id=\"113\", container=\"kube-multus\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545424240
Jul 17 03:11:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:54.167Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_local_link_integrity_errors_total\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_6\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545443391
Jul 17 03:11:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:54.227Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", phase=\"Pending\", pod=\"ipmi-exporter-6f8464dd94-ws8dm\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"e499bad4-c3e0-46d2-8043-b45c617d5590\"}" timestamp=1689545444478
Jul 17 03:11:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:54.231Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_fifo_total\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth3449d361\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545448328
Jul 17 03:11:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:54.249Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.grpc.v1.cri-sandboxes-d6129ebaf9475ba7ac6ba82f55bbd4e3053df224a620c6de773313eff8678572-shm.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689545440884
Jul 17 03:11:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:54.442Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", container=\"prometheus-alertmanager\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"cadvisor\", failure_type=\"pgmajfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"container\"}" timestamp=1689545447785
Jul 17 03:11:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:54.638Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_reservation_limit_bytes\", asset=\"pcap-tsgx11\", asset_id=\"115\", container=\"dign-server-web\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545450039
Jul 17 03:11:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:54.745Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx08\", asset_id=\"65\", container=\"telegraf-security\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Discard\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689545422069
Jul 17 03:11:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:54.772Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_annotations\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"systemd-exporter-7f549d9485\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545444478
Jul 17 03:11:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:54.806Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx12\", asset_id=\"113\", cpu=\"49\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545443391
Jul 17 03:11:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:54.809Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx08\", asset_id=\"65\", cpu=\"40\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545448328
Jul 17 03:11:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:54.834Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_labels\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"tsg-traffic-engine-numa-0-firewall\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545448663
Jul 17 03:11:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:54.836Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_max_fds\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"tuned.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689545452135
Jul 17 03:11:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:54.973Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cpu=\"80\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545439409
Jul 17 03:11:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:55.249Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_writes_bytes_total\", asset=\"msh-tsgx08\", asset_id=\"65\", container=\"mrapm-device\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545424329
Jul 17 03:11:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:55.329Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_up\", asset=\"pcap-dellr740-dt002\", asset_id=\"33\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"em3\", endpoint=\"NC-Gohangout-pcap-dellr740-dt002\", endpoint_id=\"173\", module=\"DC-Gohangout\", module_id=\"35\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545452666
Jul 17 03:11:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:55.373Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_bytes_total\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth25a03679\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545448328
Jul 17 03:11:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:55.375Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_packets_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"cnibr0\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545439409
Jul 17 03:11:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:55.418Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-podfed9548b_8974_4c11_aec7_27cda4c82f15.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"slice\"}" timestamp=1689545445496
Jul 17 03:11:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:55.642Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_virtual_memory_max_bytes\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"k3s.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689545452135
Jul 17 03:11:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:55.684Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_packets_dropped_total\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"cadvisor\", interface=\"ens2f0np0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545432275
Jul 17 03:11:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:55.709Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", cpu=\"82\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545449337
Jul 17 03:11:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:55.744Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_drop_total\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"vethf05dc1bb\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545448328
Jul 17 03:11:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:55.754Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", cpu=\"53\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545449337
Jul 17 03:11:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:55.811Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"msh-tsgx08\", asset_id=\"65\", cpu=\"13\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545448328
Jul 17 03:11:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:55.832Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx12\", asset_id=\"113\", cpu=\"88\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545443391
Jul 17 03:11:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:55.851Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"msh-tsgx12\", asset_id=\"113\", cpu=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545443391
Jul 17 03:11:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:55.865Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"nvmefc-boot-connections.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"oneshot\", type=\"service\"}" timestamp=1689545440884
Jul 17 03:11:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:55.886Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"tmpfs\", name=\"run-k3s-containerd-io.containerd.grpc.v1.cri-sandboxes-018432426ad7da9b65c9766f5b54868f50d1d8fca48fa6b88012c06d481af1eb-shm.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689545451179
Jul 17 03:11:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:55.910Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"clickhouse_block_discard_ops_sda\", asset=\"pcap-dellr740-dt003\", asset_id=\"35\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"NC-Clickhouse-pcap-dellr740-dt003\", endpoint_id=\"85\", module=\"NC-Clickhouse\", module_id=\"29\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545453231
Jul 17 03:11:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:55.917Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"clickhouse_cpu_frequency_m_hz_27\", asset=\"pcap-dellr740-dt003\", asset_id=\"35\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"NC-Clickhouse-pcap-dellr740-dt003\", endpoint_id=\"85\", module=\"NC-Clickhouse\", module_id=\"29\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545453231
Jul 17 03:11:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:56.193Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db92\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545450108
Jul 17 03:11:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:56.380Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"abrt-vmcore.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689545451179
Jul 17 03:11:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:56.405Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_info\", asset=\"pcap-tsgx07\", asset_id=\"85\", container=\"local-path-provisioner\", container_id=\"containerd://861ed970b6812a52f4de3720397895b948a3cda1667f7f559e790f7ef90db6d8\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", image=\"docker.io/rancher/local-path-provisioner:v0.0.24\", image_id=\"sha256:b29384aeb4b13e047448ccfd312c52b4d023abcbbaafcab174293a97821dddb0\", image_spec=\"rancher/local-path-provisioner:v0.0.24\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"local-path-provisioner-957fdf8bc-5lccq\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"3a6dd972-267f-421e-975e-5b26c18a80ad\"}" timestamp=1689545442058
Jul 17 03:11:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:56.423Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-virtual-net-veth25a03679.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689545452135
Jul 17 03:11:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:56.724Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-tsg-traffic-engine-numa-0-clqgr\", project=\"TSG-OS\", project_id=\"7\", reason=\"NodeLost\", service=\"prometheus-kube-state-metrics\", uid=\"1ea39cdb-8ff7-4292-b608-b56268d85d12\"}" timestamp=1689545442058
Jul 17 03:11:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:56.764Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_errs_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"vethe9e8153e\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545439409
Jul 17 03:11:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:57.167Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_errors_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"cadvisor\", interface=\"ens3f0np0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545450184
Jul 17 03:11:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:57.228Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_annotations\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"cm-policy-local-cache-redis-master-69967b5879-sjdjw\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"5c522fdc-927c-4897-9ae6-09e4fb08b6fe\"}" timestamp=1689545453389
Jul 17 03:11:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:57.316Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"105\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545445883
Jul 17 03:11:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:57.335Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_client_recent_max_output_buffer_bytes\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545446475
Jul 17 03:11:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:57.351Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_netstat_Udp_InErrors\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545445883
Jul 17 03:11:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:57.595Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"72\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545445883
Jul 17 03:11:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:57.669Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"snmpd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689545443372
Jul 17 03:11:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:57.814Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-ttyS1.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689545443372
Jul 17 03:11:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:57.895Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", container=\"dign-server-web\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"iowaiting\"}" timestamp=1689545444214
Jul 17 03:11:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:57.932Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_errs_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens3f0np0\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545445883
Jul 17 03:11:58 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:58.139Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", container=\"metrics-server\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"cadvisor\", failure_type=\"pgmajfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"container\"}" timestamp=1689545432550
Jul 17 03:11:58 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:58.202Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_mapped_file\", asset=\"pcap-tsgx03\", asset_id=\"73\", container=\"telegraf-security\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689545441497
Jul 17 03:11:58 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:58.276Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_cpu_period\", asset=\"pcap-tsgx03\", asset_id=\"73\", container=\"prometheus-server\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545451563
Jul 17 03:11:58 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:58.379Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_terminated_reason\", asset=\"pcap-tsgx03\", asset_id=\"73\", container=\"helm\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-cm-policy-local-cache-7cvjd\", project=\"TSG-OS\", project_id=\"7\", reason=\"Completed\", service=\"prometheus-kube-state-metrics\", uid=\"1d88af86-390b-43f1-be04-18028cd32b55\"}" timestamp=1689545453389
Jul 17 03:11:58 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:58.896Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"38\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545445883
Jul 17 03:11:58 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:58.904Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_reads_bytes_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", container=\"dign-server-dns\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545439211
Jul 17 03:11:58 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:58.962Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-7eb22f8239232baff63e663a09168632214f7beb47dcdcbd21f831e822cf1996-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689545443372
Jul 17 03:11:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:59.245Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_terminated\", asset=\"pcap-tsgx03\", asset_id=\"73\", container=\"redis\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"cm-policy-local-cache-redis-master-69967b5879-sjdjw\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"5c522fdc-927c-4897-9ae6-09e4fb08b6fe\"}" timestamp=1689545453389
Jul 17 03:11:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:59.453Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"107\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545445883
Jul 17 03:11:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:59.608Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth88dfc5f3\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545445883
Jul 17 03:11:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:59.801Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_heap_idle_bytes\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545445071
Jul 17 03:11:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:59.822Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_compressed_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens6f1np1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545445883
Jul 17 03:11:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:11:59.860Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"72\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545445883
Jul 17 03:12:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:00.992Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_multicast_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth844980e5\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545449083
Jul 17 03:12:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:00.999Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-55b23504\\\\x2d098e\\\\x2d4605\\\\x2d9a10\\\\x2d91ab01c5f1f9-volume\\\\x2dsubpaths-config\\\\x2dvolume-telegraf\\\\x2dsecurity-1.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689545444879
Jul 17 03:12:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:01.029Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx01\", asset_id=\"51\", container=\"telegraf\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Write\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689545447383
Jul 17 03:12:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:01.200Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_arp_entries\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"bond0\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545449083
Jul 17 03:12:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:01.216Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"9\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545449083
Jul 17 03:12:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:01.474Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_size_bytes\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"shm\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/b38bf4127a0eab7432569ab9630c34b580958f0e81207d3a7c1f66e8a2177ce7/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545449083
Jul 17 03:12:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:01.493Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_up\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"vethb97771d9\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545449083
Jul 17 03:12:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:01.774Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_replication_backlog_bytes\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545445071
Jul 17 03:12:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:01.806Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_mtu_bytes\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth0943ee2a\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545449083
Jul 17 03:12:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:01.959Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_created\", asset=\"msh-tsgx01\", asset_id=\"51\", configmap=\"shaping-tsg-traffic-engine-numa-1\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545447425
Jul 17 03:12:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:01.976Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_memory_used_rss_bytes\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545445071
Jul 17 03:12:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:01.997Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-4bb8d6b7515c62ace94ed32f8051d4ba03014412f0922b77f03a02eb3ce93865-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689545444879
Jul 17 03:12:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:02.055Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_speed_bytes\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth844980e5\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545449083
Jul 17 03:12:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:02.073Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:00-0000:00:01.1-0000:01:00.0-infiniband_mad-umad0.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689545444879
Jul 17 03:12:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:02.585Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_drop_total\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth2ff062e3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545458318
Jul 17 03:12:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:02.638Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"home.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689545448447
Jul 17 03:12:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:03.059Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_status_failed\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", job_name=\"helm-install-prometheus\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545451412
Jul 17 03:12:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:03.092Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx02\", asset_id=\"63\", cpu=\"35\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545458318
Jul 17 03:12:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:03.322Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"net_conntrack_dialer_conn_failed_total\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", dialer_name=\"25\", instance=\"10.10.20.159:10090\", reason=\"timeout\", type=\"global\"}" timestamp=1689545462989
Jul 17 03:12:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:03.541Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_inodes_total\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/sys/fs/cgroup\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545447996
Jul 17 03:12:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:03.550Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_mapped_file\", asset=\"msh-tsgx02\", asset_id=\"63\", container=\"prometheus-server\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545446176
Jul 17 03:12:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:03.577Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_annotations\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"prometheus-alertmanager\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545451412
Jul 17 03:12:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:03.615Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"sleeping\"}" timestamp=1689545451870
Jul 17 03:12:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:03.665Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_bytes_total\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"lo\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545458318
Jul 17 03:12:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:03.863Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"prometheus_target_scrape_pool_sync_total\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", instance=\"10.10.20.159:10090\", scrape_job=\"asset_ping_119\", type=\"global\"}" timestamp=1689545462989
Jul 17 03:12:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:03.944Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"prometheus_target_sync_length_seconds\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", instance=\"10.10.20.159:10090\", quantile=\"0.99\", scrape_job=\"47\", type=\"global\"}" timestamp=1689545462989
Jul 17 03:12:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:04.521Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_spec_strategy_rollingupdate_max_unavailable\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"dashboard-metrics-scraper\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545451412
Jul 17 03:12:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:04.545Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_info\", asset=\"msh-tsgx02\", asset_id=\"63\", created_by_kind=\"ReplicaSet\", created_by_name=\"prometheus-kube-state-metrics-6b768c7bf8\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", host_ip=\"10.10.20.162\", host_network=\"false\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", node=\"localhost\", nz_agent_id=\"3\", pod=\"prometheus-kube-state-metrics-6b768c7bf8-rlg7j\", pod_ip=\"10.42.0.152\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"48fe8066-b792-469f-a7b4-f5bfac9a3865\"}" timestamp=1689545451412
Jul 17 03:12:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:04.566Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2duuid-0440\\\\x2dBF8B.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689545448447
Jul 17 03:12:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:04.619Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_load_average_10s\", asset=\"msh-tsgx02\", asset_id=\"63\", container=\"telegraf\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689545448692
Jul 17 03:12:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:04.671Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"msh-tsgx02\", asset_id=\"63\", condition=\"Available\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"ipmi-exporter\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"false\"}" timestamp=1689545451412
Jul 17 03:12:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:04.997Z caller=dedupe.go:112 component=remote level=info remote_name=52db6f url=http://10.10.10.159:10090/nz-agent/prometheus/proxy/api/v1/write msg="Dropped sample for series that was not explicitly dropped via relabelling" ref=4436650
Jul 17 03:12:05 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:05.275Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_frequency_min_hertz\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", cpu=\"3\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-dt001\", endpoint_id=\"225\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545462987
Jul 17 03:12:05 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:05.306Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"net_conntrack_dialer_conn_failed_total\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", dialer_name=\"397\", instance=\"10.10.20.159:10090\", reason=\"timeout\", type=\"global\"}" timestamp=1689545462989
Jul 17 03:12:05 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:05.320Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"prometheus_http_request_duration_seconds_bucket\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", handler=\"/api/v1/metadata\", instance=\"10.10.20.159:10090\", le=\"+Inf\", type=\"global\"}" timestamp=1689545462989
Jul 17 03:12:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:06.668Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-infiniband-umad9.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689545453582
Jul 17 03:12:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:07.366Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_metadata_resource_version\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.tsg-traffic-engine-numa-0.v3\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545461227
Jul 17 03:12:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:07.603Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_threads_max\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689545454850
Jul 17 03:12:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:07.624Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_waiting\", asset=\"twa-tsgx01\", asset_id=\"91\", container=\"helm\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-prometheus-6pl4b\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"bd05c669-012d-49fa-8d25-3fc0dd995067\"}" timestamp=1689545461227
Jul 17 03:12:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:07.645Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_ips\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", ip=\"10.42.0.159\", ip_family=\"4\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", pod=\"kubernetes-dashboard-66dd5b456-hd6nt\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"37167b92-d06b-4d10-8d55-65300f19e016\"}" timestamp=1689545461227
Jul 17 03:12:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:07.676Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"twa-tsgx01\", asset_id=\"91\", condition=\"false\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-cm-policy-local-cache-x2s95\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"a269b532-737c-464a-963c-78a3bfa6b3d6\"}" timestamp=1689545461227
Jul 17 03:12:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:07.698Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_annotations\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"kubernetes-dashboard-key-holder\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545461227
Jul 17 03:12:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:07.842Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_packets_dropped_total\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"cadvisor\", interface=\"eno33np0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545458700
Jul 17 03:12:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:07.860Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"initrd-fs.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"target\"}" timestamp=1689545453582
Jul 17 03:12:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:07.879Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"var-lib-kubelet-pods-7ce93833\\\\x2d4c85\\\\x2d4a59\\\\x2dbe25\\\\x2d1ba1b6f91e91-volumes-kubernetes.io\\\\x7esecret-cm\\\\x2dcache\\\\x2dextra\\\\x2dsecrets.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689545453582
Jul 17 03:12:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:07.933Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_persistentvolume_status_phase\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", persistentvolume=\"pvc-9895065e-9211-44f6-bb57-487c596ff549\", phase=\"Pending\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545461227
Jul 17 03:12:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:08.215Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_commands_failed_calls_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cmd=\"slowlog\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545458290
Jul 17 03:12:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:08.229Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"twa-tsgx01\", asset_id=\"91\", condition=\"unknown\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-configmap-reloader-q5vrx\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"ad6201ff-fd8c-4137-9755-b2e01f9d766a\"}" timestamp=1689545461227
Jul 17 03:12:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:08.311Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_complete\", asset=\"twa-tsgx01\", asset_id=\"91\", condition=\"true\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", job_name=\"helm-install-ipmi-exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545461227
Jul 17 03:12:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:08.331Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_state_started\", asset=\"twa-tsgx01\", asset_id=\"91\", container=\"redis\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"cm-policy-local-cache-redis-master-69967b5879-qz5fq\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"7ce93833-4c85-4a59-be25-1ba1b6f91e91\"}" timestamp=1689545461227
Jul 17 03:12:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:08.417Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_compressed_total\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth3c64cde3\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545463910
Jul 17 03:12:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:08.436Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_textfile_scrape_error\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545463910
Jul 17 03:12:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:08.472Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"30\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545463910
Jul 17 03:12:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:08.498Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-cnibr0.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689545453582
Jul 17 03:12:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:08.627Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"cortex_response_message_bytes_sum\", agent_name=\"msh-kvm-nz001\", application=\"cortex\", datacenter=\"MSH\", instance=\"127.0.0.1:10093\", method=\"gRPC\", route=\"/frontend.Frontend/Process\"}" timestamp=1689545467263
Jul 17 03:12:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:08.998Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:00-0000:00:01.1-0000:01:00.1-infiniband_mad-umad1.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689545453582
Jul 17 03:12:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:11.008Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_scanCount\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"default\", project=\"OLAP\", project_id=\"3\", region=\"2e1f4bf7efca655438213e8ba2c8c3a7\", table=\"index_partfile_default\", type=\"index\"}" timestamp=1689545468804
Jul 17 03:12:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:11.181Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_compactionsQueuedCount\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"proxy_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"d74bb32ffdc6517c1bcb927023c29bee\", table=\"index_partfile_proxy_hos_bucket\", type=\"index\"}" timestamp=1689545468804
Jul 17 03:12:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:11.445Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_maxStoreFileAge\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", namespace=\"tsg_galaxy\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"6554a9b2f72daa65bcb15493980cd1f1\", table=\"relation_framedip_account\", type=\"data\"}" timestamp=1689545470149
Jul 17 03:12:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:12.426Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_4_metric_scanTime_99_9th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545470149
Jul 17 03:12:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:12.446Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_193_1_metric_majorCompactionInputSize_25th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545470149
Jul 17 03:12:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:12.451Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_replicaid\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"knowledge_base_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"bf2a63dbae3b7c03de972a5d882a0894\", table=\"index_filename_knowledge_base_hos_bucket\", type=\"index\"}" timestamp=1689545470149
Jul 17 03:12:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:12.451Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"java_lang_MemoryPool_PeakUsage_max\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"Metaspace\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545470149
Jul 17 03:12:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:12.464Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_3_metric_getTime_98th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545470149
Jul 17 03:12:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:12.531Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_filteredReadRequestCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"proxy_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"2097b60eaa4ffea308d53e366e4242fb\", table=\"index_filename_proxy_hos_bucket\", type=\"index\"}" timestamp=1689545470149
Jul 17 03:12:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:12.550Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_194_2_metric_compactionInputFileCount_median\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545470149
Jul 17 03:12:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:12.681Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_putCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"troubleshooting_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"b3f77aa4a62fb97879f86a184bd53a12\", table=\"index_partfile_troubleshooting_hos_bucket\", type=\"index\"}" timestamp=1689545470149
Jul 17 03:12:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:12.696Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_scanCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"knowledge_base_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"d7545eddd6684b532e594f03edaf2b47\", table=\"index_time_knowledge_base_hos_bucket\", type=\"index\"}" timestamp=1689545470149
Jul 17 03:12:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:12.716Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_TASK_metric_appendTime_95th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545470149
Jul 17 03:12:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:12.720Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_heap_inuse_bytes\", asset=\"twa-dellr740-dt002\", asset_id=\"39\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"NC-Gohangout-twa-dellr740-dt002\", endpoint_id=\"179\", module=\"DC-Gohangout\", module_id=\"35\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545471525
Jul 17 03:12:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:12.722Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_default_metric_incrementTime_mean\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545470149
Jul 17 03:12:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:12.730Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_193_1_metric_averageRegionSize\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545470149
Jul 17 03:12:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:15.750Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_avail_bytes\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"shm\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/2e122063ee2c201eace04f25ac4f6d02502e3258d1a3613479961720dee5e2b1/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545461230
Jul 17 03:12:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:15.782Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-64082a69\\\\x2d3be2\\\\x2d4b5e\\\\x2d9e3a\\\\x2dc0da16576d9e-volume\\\\x2dsubpaths-config\\\\x2dvolume-firewall-1.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689545467046
Jul 17 03:12:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:15.853Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-pod64082a69_3be2_4b5e_9e3a_c0da16576d9e.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"slice\"}" timestamp=1689545467046
Jul 17 03:12:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:15.948Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2dpath-pci\\\\x2d0000:81:00.0\\\\x2dscsi\\\\x2d0:2:0:0\\\\x2dpart3.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689545467046
Jul 17 03:12:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:15.963Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"mrzcpd_hugepages_setup.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"service\"}" timestamp=1689545467046
Jul 17 03:12:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:15.993Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545461230
Jul 17 03:12:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:16.006Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"ext4\", name=\"data-var-lib-kubelet-pods-64082a69\\\\x2d3be2\\\\x2d4b5e\\\\x2d9e3a\\\\x2dc0da16576d9e-volume\\\\x2dsubpaths-telegraf-telegraf-0.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689545467046
Jul 17 03:12:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:16.012Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"122\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545461230
Jul 17 03:12:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:16.039Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-sysctl.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"oneshot\", type=\"service\"}" timestamp=1689545467046
Jul 17 03:12:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:16.055Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_drop_total\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"virtio_dign_s\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545461230
Jul 17 03:12:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:16.132Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_observed_generation\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"tsg-traffic-engine-numa-0-firewall\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545468076
Jul 17 03:12:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:16.161Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_ready\", asset=\"twa-tsgx07\", asset_id=\"103\", container=\"coredns\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"coredns-65dfb4fcf4-ss5kl\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"d909c7f6-8616-4c21-913b-225bb207a2ea\"}" timestamp=1689545468076
Jul 17 03:12:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:16.191Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_annotations\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"shaping-announce-port-tsg-traffic-engine-numa-0\", uid=\"2f008c96-0519-4007-ab83-c26d965f7a9c\"}" timestamp=1689545468076
Jul 17 03:12:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:16.229Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_up\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens3f0np0\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545461230
Jul 17 03:12:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:16.271Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_io_time_weighted_seconds_total\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda2\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545465347
Jul 17 03:12:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:16.290Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"34\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545461230
Jul 17 03:12:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:16.409Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"22\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545461230
Jul 17 03:12:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:16.433Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"0\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545461230
Jul 17 03:12:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:16.450Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_errors_total\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"cadvisor\", interface=\"ens2f0np0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545466013
Jul 17 03:12:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:16.457Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_created\", asset=\"twa-tsgx07\", asset_id=\"103\", configmap=\"kube-root-ca.crt\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-node-lease\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545468076
Jul 17 03:12:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:16.545Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"tmpfs\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/var/lib/kubelet/pods/030260b7-67cb-4efd-9b0b-ed9f7bff5dfa/volumes/kubernetes.io~projected/kube-api-access-6pwdl\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545461230
Jul 17 03:12:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:16.547Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"rc-local.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689545467046
Jul 17 03:12:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:16.573Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-f68561d2\\\\x2d9434\\\\x2d4bd0\\\\x2d905f\\\\x2d4bbeef0b0682-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2dw6lbq.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689545467046
Jul 17 03:12:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:16.594Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"117\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545461230
Jul 17 03:12:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:16.675Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_changes_total\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens4f2np2\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545461230
Jul 17 03:12:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:16.704Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:80-0000:80:01.1-0000:81:00.0-host0-target0:2:0-0:2:0:0-block-sda.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689545467046
Jul 17 03:12:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:16.726Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_created\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.tsg-traffic-engine-numa-1.v1\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545468076
Jul 17 03:12:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:16.859Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"23\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545461230
Jul 17 03:12:17 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:17.026Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_packets_total\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethd053ad9b\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545461230
Jul 17 03:12:17 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:17.048Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"115\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545461230
Jul 17 03:12:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:20.345Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_symbol_error_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"mlx5_1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545468263
Jul 17 03:12:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:20.347Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-traefik-crd-tpdvt\", project=\"TSG-OS\", project_id=\"7\", reason=\"NodeAffinity\", service=\"prometheus-kube-state-metrics\", uid=\"48cb73fe-f319-4cb6-948c-44756cef01f8\"}" timestamp=1689545473828
Jul 17 03:12:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:20.357Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"26\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545468263
Jul 17 03:12:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:20.407Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-udev-settle.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"service\"}" timestamp=1689545470657
Jul 17 03:12:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:20.433Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"116\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545468263
Jul 17 03:12:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:20.444Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_commands_failed_calls_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cmd=\"zremrangebyscore\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545474840
Jul 17 03:12:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:20.447Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"tmpfs\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/data/var/lib/kubelet/pods/efdb7e93-9e91-4d51-b008-336f3a7336fb/volumes/kubernetes.io~projected/kube-api-access-shnml\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545468263
Jul 17 03:12:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:20.455Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-bc823522\\\\x2d8a2a\\\\x2d46b8\\\\x2d99b2\\\\x2d927c278ba36b-volumes-kubernetes.io\\\\x7esecret-config.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689545470657
Jul 17 03:12:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:20.484Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_protocol_type\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"vethc4731574\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545468263
Jul 17 03:12:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:20.612Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-traefik-b7tft\", project=\"TSG-OS\", project_id=\"7\", reason=\"NodeAffinity\", service=\"prometheus-kube-state-metrics\", uid=\"f8334eab-55b4-4187-9700-4ad2897e2d91\"}" timestamp=1689545473828
Jul 17 03:12:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:20.634Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_fully_labeled_replicas\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"prometheus-pushgateway-bd476bcf9\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545473828
Jul 17 03:12:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:21.151Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_up_changes_total\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"virtio_dign_c\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545477612
Jul 17 03:12:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:21.163Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"shm\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/45784156ce269581b60a89b1dc8a9e4c4ffa5e52b9b68d34ffa69b7a23bd2ba9/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545477612
Jul 17 03:12:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:21.181Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_queue_length\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth0ae4eb9d\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545477612
Jul 17 03:12:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:21.197Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2dlabel-TGS\\\\x2dOS\\\\x2dSYSROOT.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689545464356
Jul 17 03:12:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:21.390Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:20-0000:20:03.1-0000:21:00.0-net-ens2f0np0.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689545464356
Jul 17 03:12:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:21.396Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_compressed_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth8d94c437\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545468263
Jul 17 03:12:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:21.413Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-087cb80f758630b8574ef9d26d91341cda0096d1c902d60974db3ae9179ab956-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689545470657
Jul 17 03:12:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:21.459Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_packets_total\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"virtio_dign_c\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545477612
Jul 17 03:12:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:21.473Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_packets_total\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethdd9c27e1\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545477612
Jul 17 03:12:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:21.489Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_memory_Inactive_anon_bytes\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545468263
Jul 17 03:12:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:21.637Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", container=\"metrics\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"stopped\"}" timestamp=1689545466425
Jul 17 03:12:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:21.772Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_sockets\", asset=\"pcap-tsgx10\", asset_id=\"89\", container=\"dign-server-web\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545466215
Jul 17 03:12:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:21.789Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"81\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545468263
Jul 17 03:12:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:21.881Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_info\", address=\"e4:3d:1a:86:92:40\", asset=\"pcap-tsgx10\", asset_id=\"89\", broadcast=\"ff:ff:ff:ff:ff:ff\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens4f0np0\", duplex=\"unknown\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operstate=\"down\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545468263
Jul 17 03:12:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:21.894Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"25\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545468263
Jul 17 03:12:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:21.957Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"pcap-tsgx10\", asset_id=\"89\", condition=\"unknown\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"traefik-64f55bb67d-v57x6\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"eb9c8310-844c-4b72-9e0c-d86e20973848\"}" timestamp=1689545473828
Jul 17 03:12:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:22.036Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"110\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545468263
Jul 17 03:12:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:22.592Z caller=dedupe.go:112 component=remote level=error remote_name=52db6f url=http://10.10.10.159:10090/nz-agent/prometheus/proxy/api/v1/write msg="non-recoverable error" count=1000 exemplarCount=0 err="server returned HTTP status 400 : out of order sample"
Jul 17 03:12:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:23.654Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_packets_total\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth6b39ce02\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545474790
Jul 17 03:12:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:23.752Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_status_completion_time\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", job_name=\"helm-install-systemd-exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545480068
Jul 17 03:12:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:23.757Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_max_usage_bytes\", asset=\"msh-tsgx09\", asset_id=\"55\", container=\"prometheus-alertmanager-configmap-reload\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545472763
Jul 17 03:12:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:23.777Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_terminated\", asset=\"msh-tsgx09\", asset_id=\"55\", container=\"dashboard-metrics-scraper\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", pod=\"dashboard-metrics-scraper-5b94897dc9-t6qds\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"9ab87c24-ae10-4bca-bc1b-c09bbe929130\"}" timestamp=1689545480068
Jul 17 03:12:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:23.781Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_labels\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-alertmanager-6df7f457b6-ct85d\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"e335f8b4-74cf-439e-9760-c54f88f85a5d\"}" timestamp=1689545480068
Jul 17 03:12:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:23.794Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Failed\", pod=\"helm-install-cm-policy-local-cache-kppzx\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"25dfbfd0-9647-4abe-b4a0-2480fdfb4995\"}" timestamp=1689545480068
Jul 17 03:12:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:23.810Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx09\", asset_id=\"55\", cpu=\"37\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545474790
Jul 17 03:12:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:23.831Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-0bf21c684b5dc9877d9ee0eac226b483f9fccbc9cba592f7e5f290343873248a.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"scope\"}" timestamp=1689545478416
Jul 17 03:12:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:23.837Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx09\", asset_id=\"55\", cpu=\"85\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545474790
Jul 17 03:12:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:23.844Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db83\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545477019
Jul 17 03:12:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:23.855Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_aof_last_cow_size_bytes\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545477019
Jul 17 03:12:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:23.874Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"scrape_duration_seconds\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_instance=\"10.42.0.154:9005\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"mrzcpd-exporter\"}" timestamp=1689545480889
Jul 17 03:12:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:23.888Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-infiniband-umad6.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689545478416
Jul 17 03:12:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:23.906Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"process_resident_memory_bytes\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"ipmi-exporter\"}" timestamp=1689545483120
Jul 17 03:12:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:23.919Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"75\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545474790
Jul 17 03:12:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:23.966Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-rdma-devices-mlx5_9:1-umad.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689545478416
Jul 17 03:12:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:23.974Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-eno33np0.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689545478416
Jul 17 03:12:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:24.005Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-e5d1d10290add0d5022101c67a41ccb9c7da13f2242f7941ff3f3b8fc1071644.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"scope\"}" timestamp=1689545478416
Jul 17 03:12:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:24.057Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.grpc.v1.cri-sandboxes-829dcc3454fe3dbf12a2bd9128aa89e682942311ec5fc93d9fa3f5166645b58e-shm.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689545478416
Jul 17 03:12:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:24.159Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"62\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545480160
Jul 17 03:12:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:24.180Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"msh-tsgx09\", asset_id=\"55\", cpu=\"109\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545474790
Jul 17 03:12:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:24.206Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"dashboard-metrics-scraper\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Total\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545477125
Jul 17 03:12:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:24.233Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"79\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545480160
Jul 17 03:12:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:24.237Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_processes\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"kubernetes-dashboard\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545470575
Jul 17 03:12:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:24.301Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"52\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545480160
Jul 17 03:12:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:24.335Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_frame_total\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth0c6b6d72\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545474790
Jul 17 03:12:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:24.361Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_created\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\", uid=\"d9100398-4037-44ea-b6e9-844ea6413f18\"}" timestamp=1689545471122
Jul 17 03:12:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:24.381Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"111\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545480160
Jul 17 03:12:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:24.396Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_packets_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", interface=\"eth0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545482829
Jul 17 03:12:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:24.412Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_info\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"prometheus-node-exporter\", container_id=\"containerd://077ad337df6aa6d2b1064c1f0e3036e1dfd0a8e71e3f3daf0635f5be98ec9e4c\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", image=\"quay.io/prometheus/node-exporter:v1.3.1\", image_id=\"sha256:1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26\", image_spec=\"quay.io/prometheus/node-exporter:v1.3.1\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-node-exporter-qmjq7\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"e8a13065-a344-44f4-a0e1-7369c47522e2\"}" timestamp=1689545471122
Jul 17 03:12:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:24.420Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"9\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545480160
Jul 17 03:12:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:24.430Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"91\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545480160
Jul 17 03:12:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:24.435Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"auditd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689545471102
Jul 17 03:12:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:24.463Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_netstat_Icmp_OutMsgs\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545480160
Jul 17 03:12:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:24.464Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_hwmon_chip_names\", asset=\"pcap-tsgx12\", asset_id=\"117\", chip=\"0000:60:03_1_0000:63:00_1\", chip_name=\"bnxt_en\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545480160
Jul 17 03:12:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:24.479Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_drop_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"vethdb91c360\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545480160
Jul 17 03:12:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:24.487Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db213\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545481795
Jul 17 03:12:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:24.603Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_address_assign_type\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth64615143\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545480160
Jul 17 03:12:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:24.716Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_usage_bytes\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"kube-multus\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545466407
Jul 17 03:12:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:24.737Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"prometheus-alertmanager\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545471122
Jul 17 03:12:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:24.755Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"103\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545480160
Jul 17 03:12:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:24.805Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-rdma-devices-mlx5_11:1-umad.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689545478416
Jul 17 03:12:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:24.811Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failcnt\", asset=\"msh-tsgx09\", asset_id=\"55\", container=\"local-path-provisioner\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545475113
Jul 17 03:12:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:24.831Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"77\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545480160
Jul 17 03:12:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:24.892Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_cpu_seconds_total\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-pod2925075f_964b_44d6_8ad8_9a4ac784d408.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"slice\"}" timestamp=1689545478416
Jul 17 03:12:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:24.912Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas_unavailable\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"ipmi-exporter\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545480068
Jul 17 03:12:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:25.060Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_info\", asset=\"pcap-tsgx12\", asset_id=\"117\", board_id=\"MT_0000000359\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"mlx5_7\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", firmware_version=\"22.28.4000\", hca_type=\"MT4125\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545480160
Jul 17 03:12:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:25.068Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_annotations\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", uid=\"ca5d09c1-ed8b-40d1-9795-1f331f9aea4c\"}" timestamp=1689545471122
Jul 17 03:12:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:25.074Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_up_changes_total\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens4f0np0\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545474790
Jul 17 03:12:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:25.087Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx09\", asset_id=\"55\", cpu=\"76\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545474790
Jul 17 03:12:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:25.121Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"127\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545480160
Jul 17 03:12:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:25.189Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"systemd-exporter-7f549d9485-qgpb4\", project=\"TSG-OS\", project_id=\"7\", reason=\"UnexpectedAdmissionError\", service=\"prometheus-kube-state-metrics\", uid=\"9379d048-15af-4f3f-ae10-5d1f0ef6e79a\"}" timestamp=1689545480068
Jul 17 03:12:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:25.483Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_colls_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens4f3np3\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545480160
Jul 17 03:12:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:26.216Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_fifo_total\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"eno36np3\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545475810
Jul 17 03:12:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:26.342Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_restart_policy\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"mrzcpd-exporter-5cf7f967cc-5s8vp\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", type=\"Always\", uid=\"e79a435c-7f48-4427-9500-7dd3dac82d83\"}" timestamp=1689545478744
Jul 17 03:12:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:26.365Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"127\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545475810
Jul 17 03:12:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:26.665Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_bytes_total\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"cadvisor\", interface=\"ens7f1np1\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545484510
Jul 17 03:12:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:26.693Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"bond0\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545475810
Jul 17 03:12:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:26.838Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"16\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545475810
Jul 17 03:12:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:27.009Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"shm\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/2bd262b7c75754b73ebff30c2617760301fe03d563fd0f7ccffb894d58b02de8/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545475810
Jul 17 03:12:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:27.107Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"twa-tsgx09\", asset_id=\"107\", condition=\"unknown\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-traefik-vw4k2\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"f01b2d19-81c4-4465-a1a9-08cbc2270574\"}" timestamp=1689545478744
Jul 17 03:12:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:27.552Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_owner\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", owner_is_controller=\"true\", owner_kind=\"ReplicaSet\", owner_name=\"kubernetes-dashboard-66dd5b456\", pod=\"kubernetes-dashboard-66dd5b456-2cjmj\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"fa19b2ec-5070-431e-b30c-dc1e010c4b86\"}" timestamp=1689545475682
Jul 17 03:12:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:27.567Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"dign-server-pf96k\", project=\"TSG-OS\", project_id=\"7\", reason=\"Shutdown\", service=\"prometheus-kube-state-metrics\", uid=\"75dc4160-da0a-4767-823a-227b8dd303c1\"}" timestamp=1689545475682
Jul 17 03:12:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:27.573Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"msh-tsgx10\", asset_id=\"69\", condition=\"unknown\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-pushgateway-bd476bcf9-jkpjd\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"7ac64d36-f972-409d-8e70-7eed1da8b973\"}" timestamp=1689545475682
Jul 17 03:12:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:27.774Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_free_bytes\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"shm\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/e608d5789ad55db57cde72ce26e5d64eb172ad8dd0caa03d940aadf5e68899c3/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545475810
Jul 17 03:12:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:27.786Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"clevis-luks-askpass.path\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"path\"}" timestamp=1689545473907
Jul 17 03:12:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:27.798Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"102\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545475810
Jul 17 03:12:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:27.829Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_terminated_reason\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"helm\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-traefik-rdmft\", project=\"TSG-OS\", project_id=\"7\", reason=\"Completed\", service=\"prometheus-kube-state-metrics\", uid=\"3daa5d94-8d7c-4d71-883e-255619668328\"}" timestamp=1689545475682
Jul 17 03:12:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:27.845Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"72\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545482882
Jul 17 03:12:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:27.867Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_fifo_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens1f1np1\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545482882
Jul 17 03:12:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:27.902Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_compressed_total\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens3f1np1\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545475810
Jul 17 03:12:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:27.903Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"127\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545482882
Jul 17 03:12:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:27.935Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"14\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545482882
Jul 17 03:12:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:27.938Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_cpu_shares\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689545485490
Jul 17 03:12:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:27.946Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_annotations\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"kube-dns\", uid=\"c771b60a-f9b4-4ea3-8e8b-dc64415a2307\"}" timestamp=1689545478744
Jul 17 03:12:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:27.958Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545475810
Jul 17 03:12:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:28.014Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_excessive_buffer_overrun_errors_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_2\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545482882
Jul 17 03:12:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:28.030Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_frame_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth20809015\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545482882
Jul 17 03:12:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:28.053Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"105\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545482882
Jul 17 03:12:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:28.073Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_mtu_bytes\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens8f0np0\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545482882
Jul 17 03:12:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:28.181Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:40-0000:40:01.1-0000:41:00.1-infiniband_mad-umad5.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689545474114
Jul 17 03:12:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:28.203Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db180\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545479446
Jul 17 03:12:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:28.219Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-infiniband-umad10.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689545474114
Jul 17 03:12:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:28.314Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_errs_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens8f1np1\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545482882
Jul 17 03:12:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:31.400Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db41\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545490141
Jul 17 03:12:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:31.450Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-1df6bd2b\\\\x2d0fea\\\\x2d4549\\\\x2d94c2\\\\x2dddadc3bbc3e4-volume\\\\x2dsubpaths-config\\\\x2dvolume-firewall-1.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689545486907
Jul 17 03:12:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:31.561Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_readonly\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"shm\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/4b292094668625108a53894581cc7c97ca49b5459d6ae9af8d67e19f6a57e76e/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545479093
Jul 17 03:12:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:31.573Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"phy_tx_packets_total\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", dev=\"virtio_dign_c\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"mrzcpd-exporter\"}" timestamp=1689545480814
Jul 17 03:12:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:31.602Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"114\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545479093
Jul 17 03:12:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:31.709Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"twa-tsgx03\", asset_id=\"95\", condition=\"unknown\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"dign-server-p4cvw\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"b3068a5c-f8e2-4c2d-89be-7648e87ab2c1\"}" timestamp=1689545490436
Jul 17 03:12:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:31.725Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"79\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545479093
Jul 17 03:12:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:31.765Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_restart_policy\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-pushgateway-bd476bcf9-jl885\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", type=\"Always\", uid=\"9a50710b-a0ce-44fe-abf8-917fc3e36a13\"}" timestamp=1689545490436
Jul 17 03:12:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:31.777Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"100\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545479093
Jul 17 03:12:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:31.781Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_oom_events_total\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"dign-server-web\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545470045
Jul 17 03:12:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:31.781Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.grpc.v1.cri-sandboxes-129d2c46d635da0e2957c38d43a2ddc0cd37dd8a73ba0372a90e3271f620c72e-shm.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689545486907
Jul 17 03:12:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:31.785Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_annotations\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.tsg-traffic-engine-numa-0.v4\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545490436
Jul 17 03:12:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:31.799Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"ipmi-exporter\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"uninterruptible\"}" timestamp=1689545467238
Jul 17 03:12:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:31.802Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"61\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545480939
Jul 17 03:12:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:31.835Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"11\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545480939
Jul 17 03:12:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:31.852Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_info\", address=\"06:15:13:f8:1c:f4\", asset=\"msh-tsgx05\", asset_id=\"67\", broadcast=\"ff:ff:ff:ff:ff:ff\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"br_dign_c\", duplex=\"unknown\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operstate=\"up\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545480939
Jul 17 03:12:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:31.864Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"configmap-reloader\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Read\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545476116
Jul 17 03:12:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:31.866Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"msh-tsgx05\", asset_id=\"67\", condition=\"true\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"ipmi-exporter-6f8464dd94-n5c4k\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"ba6ac3db-3bfc-4401-9efb-cfebf9d6d981\"}" timestamp=1689545484068
Jul 17 03:12:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:31.883Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_threads_max\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"coredns\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545480383
Jul 17 03:12:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:31.891Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-1d72bbea\\\\x2d870a\\\\x2d432c\\\\x2db71c\\\\x2d26fca67b7d6c-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2djx72k.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689545481015
Jul 17 03:12:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:31.905Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_start_time\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-traefik-crd-zttd2\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"e35afc00-e38e-48a0-ae9a-3eb25b74a58c\"}" timestamp=1689545479276
Jul 17 03:12:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:31.909Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_usage_seconds_total\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"prometheus-alertmanager-configmap-reload\", cpu=\"total\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545461836
Jul 17 03:12:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:31.914Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_labels\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"prometheus-server-6ff784d758\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545490436
Jul 17 03:12:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:31.917Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_created\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-1-firewall-6d869d4b4c\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545484068
Jul 17 03:12:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:31.921Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"102\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545480791
Jul 17 03:12:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:31.924Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_metadata_resource_version\", asset=\"msh-tsgx11\", asset_id=\"111\", configmap=\"extension-apiserver-authentication\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545479276
Jul 17 03:12:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:31.925Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas_updated\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"cm-policy-local-cache-redis-master\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545490436
Jul 17 03:12:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:31.934Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:60-0000:60:03.1-0000:63:00.3-net-eno36np3.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689545486907
Jul 17 03:12:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:31.937Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_name_assign_type\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth3448a07b\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545480791
Jul 17 03:12:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:31.941Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_init_container_status_terminated\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"firewall-init\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", pod=\"tsg-traffic-engine-numa-0-firewall-99cddfc69-25wpt\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"abf27d03-a067-448d-81a8-11df769a2208\"}" timestamp=1689545479276
Jul 17 03:12:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:31.949Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_name_assign_type\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens7f0np0\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545480791
Jul 17 03:12:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:31.952Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sysstat-summary.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"oneshot\", type=\"service\"}" timestamp=1689545482644
Jul 17 03:12:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:31.968Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_carrier_total\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth17e7614b\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545480791
Jul 17 03:12:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:31.982Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link_mode\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"eno2\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545480939
Jul 17 03:12:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:31.983Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db105\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545490141
Jul 17 03:12:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:32.000Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_bytes_total\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"vethc8e6a12a\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545480939
Jul 17 03:12:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:32.019Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"dign-client\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Total\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545478606
Jul 17 03:12:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:32.067Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sysstat-collect.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"oneshot\", type=\"service\"}" timestamp=1689545486907
Jul 17 03:12:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:32.083Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"proc-sys-fs-binfmt_misc.automount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"automount\"}" timestamp=1689545486907
Jul 17 03:12:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:32.137Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", phase=\"Pending\", pod=\"systemd-exporter-7f549d9485-mvhn4\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"ee102366-563a-4696-914f-30648a2f4c93\"}" timestamp=1689545490436
Jul 17 03:12:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:32.139Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"lb-tcp-80\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", failure_type=\"pgfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"hierarchy\"}" timestamp=1689545475737
Jul 17 03:12:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:32.174Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_address_assign_type\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens6f0np0\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545479093
Jul 17 03:12:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:32.344Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-ba6ac3db\\\\x2d3bfc\\\\x2d4401\\\\x2d9efb\\\\x2dcfebf9d6d981-volumes-kubernetes.io\\\\x7esecret-config.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689545481015
Jul 17 03:12:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:32.378Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_multicast_total\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens1f0np0\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545480939
Jul 17 03:12:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:32.405Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-a8081ba346d7b92ef4d72e84a95bbd70f93ed80b64ebc68b6ba44a14693796b9-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689545481015
Jul 17 03:12:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:32.417Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db253\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545490141
Jul 17 03:12:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:32.441Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_virtual_memory_bytes\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"snmptrapd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689545486907
Jul 17 03:12:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:32.528Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"metrics-server\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", failure_type=\"pgfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"hierarchy\"}" timestamp=1689545472853
Jul 17 03:12:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:32.560Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"30\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545479093
Jul 17 03:12:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:32.564Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_errors_total\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", interface=\"ens3f0np0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545481622
Jul 17 03:12:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:32.583Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"27\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545479093
Jul 17 03:12:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:32.588Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"msh-tsgx11\", asset_id=\"111\", condition=\"Progressing\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"prometheus-alertmanager\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"false\"}" timestamp=1689545479276
Jul 17 03:12:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:32.612Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_bytes_total\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"eno2\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545479093
Jul 17 03:12:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:32.695Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_exporter_scrape_duration_seconds_count\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545476711
Jul 17 03:12:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:32.750Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas_unavailable\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"metrics-server\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545490436
Jul 17 03:12:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:32.808Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_waiting\", asset=\"twa-tsgx03\", asset_id=\"95\", container=\"helm\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-prometheus-mshnr\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"6fb2bd33-bd15-4e41-9452-847ce05e8c34\"}" timestamp=1689545490436
Jul 17 03:12:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:32.834Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_owner\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", owner_is_controller=\"true\", owner_kind=\"Job\", owner_name=\"helm-install-tsg-traffic-engine-numa-1\", pod=\"helm-install-tsg-traffic-engine-numa-1-vnr97\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"655855cf-bcb1-4cce-bea6-d09a402d65d3\"}" timestamp=1689545490436
Jul 17 03:12:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:32.856Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_changes_total\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens8f0np0\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545480791
Jul 17 03:12:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:32.926Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-hwdb-update.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689545482644
Jul 17 03:12:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:32.937Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"scrape_series_added\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_instance=\"localhost\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545482009
Jul 17 03:12:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:32.971Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_usage_bytes\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"dign-server-web\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545470045
Jul 17 03:12:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:32.994Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-netns-cni\\\\x2d66aeb84b\\\\x2de665\\\\x2d5697\\\\x2d63c9\\\\x2d90d37fa35475.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689545482644
Jul 17 03:12:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:33.015Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"var-lib-kubelet-pods-3c517891\\\\x2dc7a5\\\\x2d4f99\\\\x2d9730\\\\x2d42d4a10fa392-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2d69tms.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689545482644
Jul 17 03:12:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:33.080Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_errs_total\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens4f1np1\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545480791
Jul 17 03:12:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:33.092Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db233\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545476711
Jul 17 03:12:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:33.116Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_state_started\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"helm\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-traefik-6j4t4\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"4c44cd75-2e3f-43c7-b5a7-ab93c02add25\"}" timestamp=1689545479276
Jul 17 03:12:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:33.195Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_frees_total\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545476711
Jul 17 03:12:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:33.337Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"msh-tsgx11\", asset_id=\"111\", condition=\"unknown\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-systemd-exporter-f9mp6\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"fa64de1f-70a4-425a-9810-f9f7d5175ae0\"}" timestamp=1689545479276
Jul 17 03:12:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:33.346Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_owner\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", owner_is_controller=\"true\", owner_kind=\"Deployment\", owner_name=\"tsg-traffic-engine-numa-1-firewall\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-1-firewall-6b6bf4679c\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545479894
Jul 17 03:12:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:33.407Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"28\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545480605
Jul 17 03:12:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:33.430Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"process_virtual_memory_bytes\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"ipmi-exporter\"}" timestamp=1689545483896
Jul 17 03:12:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:33.437Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_address_assign_type\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens4f2np2\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545480605
Jul 17 03:12:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:33.441Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_metadata_resource_version\", asset=\"msh-tsgx11\", asset_id=\"111\", configmap=\"chart-content-traefik-crd\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545479276
Jul 17 03:12:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:33.462Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"62\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545480605
Jul 17 03:12:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:33.465Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"28\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545480791
Jul 17 03:12:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:33.488Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_readonly\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"shm\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/d92ddf9eb599bea6ea57485be448ea0a442db646b5d3fb2c1b248b41ddd3d2f7/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545480791
Jul 17 03:12:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:33.500Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"102\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545480605
Jul 17 03:12:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:33.560Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_packets_total\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth17e7614b\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545480791
Jul 17 03:12:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:33.617Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"8\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545479093
Jul 17 03:12:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:33.653Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"29\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545480791
Jul 17 03:12:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:33.705Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_cache\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689545478085
Jul 17 03:12:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:33.775Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"10\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545480791
Jul 17 03:12:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:33.855Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_bytes_total\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth17383320\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545480605
Jul 17 03:12:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:33.902Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_fifo_total\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth45bb7033\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545480605
Jul 17 03:12:37 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:37.677Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"jvm_threads_states_threads\", application=\"HosServiceApplication\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"Galaxy-hos-service-msh-dellr740-dt001\", endpoint_id=\"213\", module=\"Galaxy-hos-service\", module_id=\"15\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", state=\"new\"}" timestamp=1689545497100
Jul 17 03:12:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:40.387Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-dellr740-dt003\", asset_id=\"41\", cpu=\"2\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"OLAP-Node-Exporter-twa-dellr740-dt003\", endpoint_id=\"261\", mode=\"irq\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"hbase_master,clickhouse_data,kafka,gohangout,zookeeper,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545498473
Jul 17 03:12:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:40.425Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"var-lib-kubelet-pods-497bfad5\\\\x2d323f\\\\x2d4476\\\\x2d8405\\\\x2df97cffdaa52c-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2djrt2z.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689545495764
Jul 17 03:12:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:40.508Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_persistentvolume_annotations\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", persistentvolume=\"pvc-49234523-0cb4-4f93-a353-6b9f8a2112c6\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545500034
Jul 17 03:12:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:40.530Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"23\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545491647
Jul 17 03:12:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:40.555Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_packets_dropped_total\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"cadvisor\", interface=\"ens6f0np0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545487294
Jul 17 03:12:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:40.706Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_avgStoreFileAge\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"default\", project=\"OLAP\", project_id=\"3\", region=\"86f7102593856c52aa071d1a2a7fdc4e\", table=\"index_partfile_default\", type=\"index\"}" timestamp=1689545499757
Jul 17 03:12:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:40.781Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_hwmon_temp_celsius\", asset=\"twa-tsgx04\", asset_id=\"97\", chip=\"pci0000:00_0000:00:19_3\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", sensor=\"temp10\", service=\"prometheus-node-exporter\"}" timestamp=1689545491647
Jul 17 03:12:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:40.819Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_bytes_total\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth3c119b31\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545491647
Jul 17 03:12:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:40.838Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-87f90b18\\\\x2d7e5c\\\\x2d4eda\\\\x2d916d\\\\x2d6770523a8590-volumes-kubernetes.io\\\\x7esecret-config.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689545495764
Jul 17 03:12:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:40.867Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Udp_Link_C2S\", app_name=\"sapp4\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", vsysId=\"1\"}" timestamp=1689545499161
Jul 17 03:12:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:40.884Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-b8dda7d4d07c0ae2ac831c5a3fb0594bc7450387a7efaff12ffa5760665b6517.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"scope\"}" timestamp=1689545495764
Jul 17 03:12:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:41.427Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"tmpfs\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/data/var/lib/kubelet/pods/8e8e0871-84d7-40cc-aa16-976ff516a351/volumes/kubernetes.io~projected/kube-api-access-r45w4\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545491647
Jul 17 03:12:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:41.461Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_packets_transmitted_total\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_8\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545491647
Jul 17 03:12:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:41.490Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_usage_bytes\", asset=\"twa-tsgx04\", asset_id=\"97\", container=\"prometheus-server\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545484809
Jul 17 03:12:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:41.499Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_hbase_table_rsgroup_metric_storeCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545499757
Jul 17 03:12:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:41.513Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_maxStoreFileAge\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"session_record_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"644b4cad8f6c8717769f49a507238a61\", table=\"index_filename_session_record_hos_bucket\", type=\"index\"}" timestamp=1689545499757
Jul 17 03:12:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:41.519Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx04\", asset_id=\"97\", container=\"prometheus-alertmanager\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Write\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545485407
Jul 17 03:12:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:41.535Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_user_seconds_total\", asset=\"twa-tsgx04\", asset_id=\"97\", container=\"prometheus-alertmanager-configmap-reload\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545471129
Jul 17 03:12:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:41.564Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_created\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-0-firewall-678489bbdb\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545500034
Jul 17 03:12:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:41.590Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_restart_policy\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", pod=\"dashboard-metrics-scraper-5b94897dc9-9t7mj\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", type=\"Always\", uid=\"e367f0c1-a70e-49ee-9c5c-4e518cf2740f\"}" timestamp=1689545500034
Jul 17 03:12:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:41.615Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_scrape_collector_duration_seconds\", asset=\"twa-tsgx04\", asset_id=\"97\", collector=\"zfs\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545491647
Jul 17 03:12:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:41.622Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"73\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545491647
Jul 17 03:12:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:41.651Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_info\", asset=\"twa-tsgx04\", asset_id=\"97\", configmap=\"chart-content-mrzcpd-exporter\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545500034
Jul 17 03:12:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:41.681Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_LOG_metric_tableReadQueryPerSecond_count\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545499757
Jul 17 03:12:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:41.844Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_hbase_table_rsgroup_metric_deleteBatchTime_max\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545499757
Jul 17 03:12:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:41.860Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_storeFileSize\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"knowledge_base_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"8fa0a64e312bab799a8b4d4d13552ff2\", table=\"index_time_knowledge_base_hos_bucket\", type=\"index\"}" timestamp=1689545499757
Jul 17 03:12:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:41.891Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_putCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"rtp_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"bb787935cad14024cd85d7f36821f37e\", table=\"index_filename_rtp_hos_bucket\", type=\"index\"}" timestamp=1689545499757
Jul 17 03:12:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:41.906Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_appendCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"assessment_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"a22b52399b9eb817bd362137cdea0347\", table=\"index_partfile_assessment_hos_bucket\", type=\"index\"}" timestamp=1689545499757
Jul 17 03:12:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:42.007Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_getCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"knowledge_base_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"497f09231cf4be63b13f8e182dca5dcd\", table=\"index_partfile_knowledge_base_hos_bucket\", type=\"index\"}" timestamp=1689545499757
Jul 17 03:12:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:42.044Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_getCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"firewall_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"c71e6dbfddbdc22935604dbfed59f870\", table=\"index_partfile_firewall_hos_bucket\", type=\"index\"}" timestamp=1689545499757
Jul 17 03:12:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:42.088Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_10_194_1_metric_compactionOutputFileCount_max\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545499757
Jul 17 03:12:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:42.597Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_packets_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"cadvisor\", interface=\"eth0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689545483893
Jul 17 03:12:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:42.623Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_labels\", asset=\"pcap-tsgx01\", asset_id=\"71\", configmap=\"cm-policy-local-cache-redis-health\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545495154
Jul 17 03:12:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:42.815Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_created\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.ipmi-exporter.v1\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545495154
Jul 17 03:12:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:43.062Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db203\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545494838
Jul 17 03:12:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:43.135Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_rss\", asset=\"pcap-tsgx01\", asset_id=\"71\", container=\"kube-state-metrics\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545493540
Jul 17 03:12:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:43.231Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-netns-cni\\\\x2dba776ff0\\\\x2db548\\\\x2d6a24\\\\x2d5eb5\\\\x2d13a3e4b9c647.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689545500671
Jul 17 03:12:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:43.289Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_up\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth2ab79c7d\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545496242
Jul 17 03:12:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:43.306Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_usage_bytes\", asset=\"pcap-tsgx01\", asset_id=\"71\", container=\"dashboard-metrics-scraper\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545488090
Jul 17 03:12:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:43.309Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-e00294199b4cf46d04122d5ecbf91c50dc28b9bd9be2ff2b2077a666c50c4b4b.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"scope\"}" timestamp=1689545500671
Jul 17 03:12:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:43.382Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_max_usage_bytes\", asset=\"pcap-tsgx01\", asset_id=\"71\", container=\"lb-tcp-443\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545492033
Jul 17 03:12:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:43.395Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"pcap-tsgx01\", asset_id=\"71\", condition=\"unknown\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"local-path-provisioner-957fdf8bc-r2qbd\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"4ff67ed4-4465-4d8e-8bc9-8a9bac607a46\"}" timestamp=1689545495154
Jul 17 03:12:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:43.448Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", cpu=\"126\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545496242
Jul 17 03:12:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:43.454Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_address_available\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_endpoint=\"kubernetes-dashboard\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545495154
Jul 17 03:12:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:43.511Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", cpu=\"90\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545496242
Jul 17 03:12:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:43.521Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_restarts_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", container=\"dashboard-metrics-scraper\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", pod=\"dashboard-metrics-scraper-5b94897dc9-pzv4x\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"0cfcc16a-d96c-4497-8810-811308fc4c9e\"}" timestamp=1689545495154
Jul 17 03:12:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:43.545Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_ready\", asset=\"pcap-tsgx01\", asset_id=\"71\", container=\"lb-tcp-80\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"svclb-traefik-7eb8511e-q59w6\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"0a82591e-01f6-43ee-9af0-f8cc0036ad27\"}" timestamp=1689545495154
Jul 17 03:12:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:43.559Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_dormant\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens6f1np1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545496242
Jul 17 03:12:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:43.901Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"mrzcpd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"notify\", type=\"service\"}" timestamp=1689545500671
Jul 17 03:12:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:43.929Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-pod9f0744bb_41c2_4a60_a05e_c390d7032c33.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"slice\"}" timestamp=1689545500671
Jul 17 03:12:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:43.949Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-burstable-pod095d5887_0ff1_468e_9463_a3765c458177.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"slice\"}" timestamp=1689545500671
Jul 17 03:12:48 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:48.587Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_2_metric_compactionInputFileCount_99th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545507439
Jul 17 03:12:48 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:48.625Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_194_1_metric_majorCompactionOutputFileCount_num_ops\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545507439
Jul 17 03:12:48 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:48.865Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_193_1_metric_compactionOutputSize_max\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545507439
Jul 17 03:12:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:49.023Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_193_1_metric_majorCompactionInputFileCount_75th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545507439
Jul 17 03:12:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:49.064Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_2_metric_compactionInputFileCount_99th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545507439
Jul 17 03:12:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:49.107Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_1_metric_appendTime_min\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545507439
Jul 17 03:12:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:49.130Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_3_metric_compactionOutputSize_95th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545507439
Jul 17 03:12:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:49.263Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_193_1_metric_majorCompactionOutputSize_mean\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545507439
Jul 17 03:12:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:49.288Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_4_metric_tableReadQueryPerSecond_mean_rate\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545507439
Jul 17 03:12:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:49.316Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_default_metric_putTime_75th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545507439
Jul 17 03:12:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:49.317Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_maxStoreFileAge\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", namespace=\"SYSTEM\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"b029925f044d19e54317007be48c4e04\", table=\"LOG\", type=\"data\"}" timestamp=1689545507439
Jul 17 03:12:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:49.372Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_lastMajorCompactionAge\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"session_record_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"f46da09c193e4f28d904c9d7e446fbc9\", table=\"index_time_session_record_hos_bucket\", type=\"index\"}" timestamp=1689545507439
Jul 17 03:12:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:49.394Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_session_record_hos_bucket_metric_majorCompactionOutputSize_99th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545507439
Jul 17 03:12:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:49.432Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_l2CacheHitCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Server\"}" timestamp=1689545507439
Jul 17 03:12:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:49.438Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_1_metric_compactionTime_98th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545507439
Jul 17 03:12:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:49.451Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_numReferenceFiles\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"troubleshooting_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"b301f632e0b40d5f825252a4bde96f7d\", table=\"index_filename_troubleshooting_hos_bucket\", type=\"index\"}" timestamp=1689545507439
Jul 17 03:12:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:49.715Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"java_lang_OperatingSystem_MaxFileDescriptorCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545507439
Jul 17 03:12:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:49.731Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_compactedCellsSize\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Server\"}" timestamp=1689545507439
Jul 17 03:12:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:49.762Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_1_metric_compactionInputSize_75th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545507439
Jul 17 03:12:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:49.780Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_2_metric_splitTime_99_9th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545507439
Jul 17 03:12:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:49.803Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_storeCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"rtp_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"b242966859aa9721f3ac6e8b17d51a4d\", table=\"index_filename_rtp_hos_bucket\", type=\"index\"}" timestamp=1689545507439
Jul 17 03:12:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:49.805Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"metrics\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Async\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545493105
Jul 17 03:12:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:49.810Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_193_1_metric_majorCompactionOutputSize_median\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545507439
Jul 17 03:12:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:49.835Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_194_3_metric_compactionOutputSize_95th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545507439
Jul 17 03:12:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:49.859Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_4_metric_compactionOutputSize_99th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545507439
Jul 17 03:12:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:49.883Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_2_metric_scanTime_99th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545507439
Jul 17 03:12:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:49.911Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_compactionsFailedCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"firewall_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"569c4ca2944511d5d885d0d8a8bc9016\", table=\"index_partfile_firewall_hos_bucket\", type=\"index\"}" timestamp=1689545507439
Jul 17 03:12:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:49.924Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_flushesQueuedCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"knowledge_base_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"a3ce43468e55ed101f6efab25c910474\", table=\"index_filename_knowledge_base_hos_bucket\", type=\"index\"}" timestamp=1689545507439
Jul 17 03:12:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:49.931Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_4_metric_deleteTime_mean\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545507439
Jul 17 03:12:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:49.940Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_cpu_user_seconds_total\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545506625
Jul 17 03:12:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:49.955Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_hbase_table_rsgroup_metric_putTime_99_9th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545507439
Jul 17 03:12:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:49.978Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_2_metric_compactionInputFileCount_75th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545507439
Jul 17 03:12:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:50.015Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_numFilesCompactedCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"b49931f67a477ad49cf2b6d69f7ce038\", table=\"firewall_hos_bucket\", type=\"data\"}" timestamp=1689545507439
Jul 17 03:12:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:50.042Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_alloc_bytes\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"ipmi-exporter\"}" timestamp=1689545504468
Jul 17 03:12:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:50.048Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"lb-tcp-443\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"sleeping\"}" timestamp=1689545479211
Jul 17 03:12:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:50.065Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx10\", asset_id=\"109\", cpu=\"0\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545501797
Jul 17 03:12:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:50.080Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_packets_received_total\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_8\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545501797
Jul 17 03:12:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:50.106Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_persistentvolume_status_phase\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", persistentvolume=\"pvc-c08dcd6c-f603-4169-ae73-c85cac6a245f\", phase=\"Bound\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545501568
Jul 17 03:12:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:50.118Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_CheckAndDelete_mean\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Server\"}" timestamp=1689545507439
Jul 17 03:12:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:50.129Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_tsg_galaxy_table_gtpc_knowledge_base_metric_getTime_75th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545507439
Jul 17 03:12:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:50.153Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_compactionsCompletedCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"e15cc3cd601fbf1a27864182364b8966\", table=\"rtp_hos_bucket\", type=\"data\"}" timestamp=1689545507439
Jul 17 03:12:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:50.158Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_194_1_metric_deleteBatchTime_90th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545507439
Jul 17 03:12:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:50.171Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_changes_total\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth2d6771bd\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545501797
Jul 17 03:12:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:50.184Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", pod=\"tsg-traffic-engine-numa-0-firewall-8c9c7fc95-6rm9z\", project=\"TSG-OS\", project_id=\"7\", reason=\"Evicted\", service=\"prometheus-kube-state-metrics\", uid=\"207ceb95-e1ef-4b7e-a587-0a6495b0f134\"}" timestamp=1689545501568
Jul 17 03:12:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:50.187Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_complete\", asset=\"twa-tsgx08\", asset_id=\"105\", condition=\"true\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", job_name=\"helm-install-traefik\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545501568
Jul 17 03:12:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:50.204Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_3_metric_majorCompactionTime_99_9th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545507439
Jul 17 03:12:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:50.213Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx10\", asset_id=\"109\", cpu=\"94\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545501797
Jul 17 03:12:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:50.217Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_up_changes_total\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethbd8ba9ba\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545501797
Jul 17 03:12:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:50.236Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-706d1113aac372b7fbb3191e0fff56a0981e18d8acf3274c04e13a2bb1ec50d1.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"scope\"}" timestamp=1689545501057
Jul 17 03:12:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:50.244Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"ipmi-exporter\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Discard\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545491211
Jul 17 03:12:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:50.267Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx10\", asset_id=\"109\", cpu=\"102\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545501797
Jul 17 03:12:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:50.292Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_address_assign_type\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"bond0\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545506709
Jul 17 03:12:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:50.294Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx10\", asset_id=\"109\", cpu=\"111\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545501797
Jul 17 03:12:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:50.311Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"twa-tsgx10\", asset_id=\"109\", cpu=\"11\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545501797
Jul 17 03:12:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:50.336Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_usage_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", container=\"dign-client\", cpu=\"total\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545502454
Jul 17 03:12:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:50.337Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_created\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"traefik-64f55bb67d\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545501568
Jul 17 03:12:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:50.349Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_bytes_total\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethcddab70d\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545501797
Jul 17 03:12:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:50.372Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"7\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545506709
Jul 17 03:12:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:50.374Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_194_4_metric_storeFileCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545507439
Jul 17 03:12:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:50.394Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:60-0000:60:03.1-0000:63:00.1-net-eno34np1.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689545501057
Jul 17 03:12:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:50.397Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_4_metric_scanTime_25th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545507439
Jul 17 03:12:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:50.400Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"72\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545506709
Jul 17 03:12:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:50.443Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_4_metric_compactionOutputFileCount_mean\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545507439
Jul 17 03:12:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:50.463Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"process_open_fds\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545507439
Jul 17 03:12:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:50.475Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"lb-tcp-443\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"iowaiting\"}" timestamp=1689545479211
Jul 17 03:12:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:50.483Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_allocator_frag_bytes\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545506625
Jul 17 03:12:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:50.500Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db234\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545506625
Jul 17 03:12:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:50.528Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:60-0000:60:03.1-0000:63:00.1-net-eno34np1.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689545501057
Jul 17 03:12:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:50.606Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_tasks_max\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"ldconfig.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689545501057
Jul 17 03:12:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:50.916Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"104\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545506709
Jul 17 03:12:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:52.516Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", container=\"prometheus-pushgateway\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"cadvisor\", failure_type=\"pgfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"hierarchy\"}" timestamp=1689545505752
Jul 17 03:12:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:52.556Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", cpu=\"59\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545509337
Jul 17 03:12:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:52.643Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_compressed_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"bond0\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545509337
Jul 17 03:12:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:52.657Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", cpu=\"78\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545509337
Jul 17 03:12:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:52.772Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", cpu=\"75\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545509337
Jul 17 03:12:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:52.855Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"overlay\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-dc43bbf15c647dd6bdac70cc8b80fc4828005aafb126dca2e19defbfccc04de0-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689545505496
Jul 17 03:12:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:52.856Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db154\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545499367
Jul 17 03:12:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:52.881Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"pcap-tsgx11\", asset_id=\"115\", condition=\"unknown\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", pod=\"tsg-traffic-engine-numa-1-firewall-6f574c7957-cv5ql\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"05cea850-febe-4ad9-a01e-de2d8c0e8a8b\"}" timestamp=1689545504478
Jul 17 03:12:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:52.904Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", cpu=\"16\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545509337
Jul 17 03:12:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:52.939Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_heap_inuse_bytes\", asset=\"pcap-dellr740-dt002\", asset_id=\"33\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"NC-Gohangout-pcap-dellr740-dt002\", endpoint_id=\"173\", module=\"DC-Gohangout\", module_id=\"35\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545512666
Jul 17 03:12:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:52.942Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_observed_generation\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"tsg-traffic-engine-numa-0-firewall\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545504478
Jul 17 03:12:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:53.083Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dnf-makecache.timer\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"timer\"}" timestamp=1689545505496
Jul 17 03:12:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:53.106Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"msh-tsgx08\", asset_id=\"65\", cpu=\"103\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545508328
Jul 17 03:12:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:53.129Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_annotations\", asset=\"pcap-tsgx11\", asset_id=\"115\", configmap=\"extension-apiserver-authentication\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545504478
Jul 17 03:12:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:53.170Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"msh-tsgx08\", asset_id=\"65\", container=\"firewall\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", state=\"uninterruptible\"}" timestamp=1689545497233
Jul 17 03:12:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:53.191Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"msh-tsgx08\", asset_id=\"65\", condition=\"true\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-tsg-traffic-engine-numa-0-rftjq\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"55e6f7ec-a65b-4932-86c3-e88f9111393a\"}" timestamp=1689545508950
Jul 17 03:12:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:53.231Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_fully_labeled_replicas\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-1-firewall-6b6bf4679c\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545508950
Jul 17 03:12:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:53.237Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_annotations\", asset=\"msh-tsgx08\", asset_id=\"65\", configmap=\"certstore-tsg-traffic-engine-numa-0\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545508950
Jul 17 03:12:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:53.257Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"network-online.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"target\"}" timestamp=1689545505496
Jul 17 03:12:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:53.266Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"38\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545508328
Jul 17 03:12:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:53.270Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_address_assign_type\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens3f0np0\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545508328
Jul 17 03:12:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:53.289Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_errs_total\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens7f1np1\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545508328
Jul 17 03:12:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:53.292Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_drop_total\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"eno34np1\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545508328
Jul 17 03:12:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:53.300Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_up\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"virtio_dign_s\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545509337
Jul 17 03:12:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:53.311Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_fifo_total\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens6f0np0\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545503391
Jul 17 03:12:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:53.325Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"vmtoolsd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"simple\", type=\"service\"}" timestamp=1689545505496
Jul 17 03:12:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:53.338Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_info\", asset=\"pcap-tsgx11\", asset_id=\"115\", configmap=\"chart-content-tsg-traffic-engine-numa-1\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545504478
Jul 17 03:12:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:53.364Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_created\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_endpoint=\"dashboard-metrics-scraper\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545504478
Jul 17 03:12:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:53.401Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2did-wwn\\\\x2d0x62cea7f0add59d002c2c317c692f9f02\\\\x2dpart4.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689545511179
Jul 17 03:12:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:53.416Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_files_free\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda5\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", fstype=\"ext4\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/data\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545509337
Jul 17 03:12:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:53.417Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_start_time_seconds\", asset=\"msh-tsgx08\", asset_id=\"65\", container=\"lb-tcp-443\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545497907
Jul 17 03:12:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:53.421Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_loading_dump_file\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545510108
Jul 17 03:12:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:53.438Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db223\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545499367
Jul 17 03:12:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:53.441Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-8ba148c1\\\\x2df25f\\\\x2d4fa4\\\\x2d82b3\\\\x2d03b9d5a8ace6-volume\\\\x2dsubpaths-config\\\\x2dvolume-firewall-1.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689545511179
Jul 17 03:12:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:53.442Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_dormant\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"cnibr0\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545508328
Jul 17 03:12:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:53.458Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_drop_total\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth8ec198c4\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545503391
Jul 17 03:12:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:53.473Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"plymouth-read-write.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"service\"}" timestamp=1689545511179
Jul 17 03:12:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:53.500Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_errors_total\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"cadvisor\", interface=\"ens4f0np0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545496840
Jul 17 03:12:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:53.537Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link_mode\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens4f2np2\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545508328
Jul 17 03:12:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:53.544Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_colls_total\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens6f1np1\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545508328
Jul 17 03:12:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:53.545Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", container=\"lb-tcp-443\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"stopped\"}" timestamp=1689545501613
Jul 17 03:12:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:53.569Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_owner\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", owner_is_controller=\"true\", owner_kind=\"Job\", owner_name=\"helm-install-tsg-traffic-engine-numa-1\", pod=\"helm-install-tsg-traffic-engine-numa-1-wpt4m\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"83ff3407-3875-4db2-bc3e-c71a4b11a80f\"}" timestamp=1689545504478
Jul 17 03:12:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:53.593Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dm-event.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689545511179
Jul 17 03:12:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:53.624Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_threads_max\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689545499169
Jul 17 03:12:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:53.763Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_observed_generation\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"ipmi-exporter\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545504478
Jul 17 03:12:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:53.811Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_NodeManager_RpcProcessingTimeNumOps\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-Yarn-msh-dellr740-dt001-NodeManager\", endpoint_id=\"351\", module=\"DC-Yarn\", module_id=\"1\", name=\"RpcActivityForPort9923\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545513514
Jul 17 03:12:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:53.858Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_swap\", asset=\"pcap-tsgx11\", asset_id=\"115\", container=\"prometheus-alertmanager-configmap-reload\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545505176
Jul 17 03:12:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:53.881Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda5\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", fstype=\"ext4\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/var/lib/kubelet/pods/93005333-2bbb-4ce3-a6b6-a5b632269832/volume-subpaths/telegraf/telegraf/0\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545509337
Jul 17 03:12:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:53.899Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_mtu_bytes\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"vethec52c02a\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545509337
Jul 17 03:12:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:53.904Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"msh-tsgx12\", asset_id=\"113\", container=\"dign-server-web\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"iowaiting\"}" timestamp=1689545490407
Jul 17 03:12:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:53.917Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2did-scsi\\\\x2dSDELL_PERC_H345_Adp_00f611881d94842c2c008cd5adf0a7ce\\\\x2dpart3.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689545505496
Jul 17 03:12:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:53.920Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_created\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"prometheus-kube-state-metrics\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545508663
Jul 17 03:12:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:53.936Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"emergency.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689545512135
Jul 17 03:12:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:53.937Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", cpu=\"30\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545509337
Jul 17 03:12:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:53.962Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-netns-cni\\\\x2d06bbd04a\\\\x2da9a8\\\\x2da01f\\\\x2dc2aa\\\\x2d02612dd72d4a.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689545505496
Jul 17 03:12:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:53.980Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_last_seen\", asset=\"pcap-tsgx07\", asset_id=\"85\", container=\"telegraf\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689545510656
Jul 17 03:12:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:53.992Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", phase=\"Failed\", pod=\"prometheus-server-6ff784d758-xb5nz\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"2b58fbb3-d9ea-4dd9-abaa-2a5af9a7f0a8\"}" timestamp=1689545502058
Jul 17 03:12:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:54.026Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_nf_conntrack_stat_invalid\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545499409
Jul 17 03:12:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:54.040Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_spec_parallelism\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", job_name=\"helm-install-ipmi-exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545504478
Jul 17 03:12:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:54.043Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-4b20d6f2e4895e567d3fc4fea227cb7eb200f7b42c5fe5405ce694ec5e4d193d.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"scope\"}" timestamp=1689545500884
Jul 17 03:12:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:54.067Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db146\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545510108
Jul 17 03:12:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:54.075Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-update-utmp-runlevel.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689545505496
Jul 17 03:12:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:54.101Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_bytes_total\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"cadvisor\", interface=\"eno2\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545496840
Jul 17 03:12:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:54.119Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"remote-fs.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"target\"}" timestamp=1689545512135
Jul 17 03:12:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:54.295Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-veth5a5e6ccb.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689545511179
Jul 17 03:12:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:54.329Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_created\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"chart-values-traefik\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545502058
Jul 17 03:12:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:54.453Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx08\", asset_id=\"65\", container=\"firewall\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Discard\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689545493450
Jul 17 03:12:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:54.533Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_mtu_bytes\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"vethae353140\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545508328
Jul 17 03:12:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:54.613Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_protocol_type\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens3f0np0\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545503391
Jul 17 03:12:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:54.812Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_usage_seconds_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", container=\"redis\", cpu=\"total\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545506586
Jul 17 03:12:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:54.858Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda5\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", fstype=\"ext4\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/var/lib/kubelet/pods/bdb49414-5fd6-4eeb-a829-ad76bdcb350b/volume-subpaths/telegraf/telegraf-security/0\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545499409
Jul 17 03:12:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:54.875Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cpu=\"36\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545499409
Jul 17 03:12:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:54.911Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_udp_queues\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", ip=\"v4\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", queue=\"rx\", service=\"prometheus-node-exporter\"}" timestamp=1689545499409
Jul 17 03:12:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:56.252Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"35\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545505883
Jul 17 03:12:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:56.277Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-virtual-net-veth464153de.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689545503372
Jul 17 03:12:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:56.292Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_spec_replicas\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"mrzcpd-exporter-5cf7f967cc\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545513389
Jul 17 03:12:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:56.303Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-ens2f0np0.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689545503372
Jul 17 03:12:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:56.336Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db96\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545506475
Jul 17 03:12:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:56.375Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-sda4.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689545503372
Jul 17 03:12:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:56.502Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_limit_bytes\", asset=\"pcap-tsgx03\", asset_id=\"73\", container=\"kube-multus\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545511563
Jul 17 03:12:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:56.632Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_dormant\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth958fa935\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545505883
Jul 17 03:12:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:56.645Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_bytes_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth6af7c83b\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545505883
Jul 17 03:12:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:56.693Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", container=\"dign-server-web\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"cadvisor\", failure_type=\"pgmajfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"container\"}" timestamp=1689545498885
Jul 17 03:12:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:56.774Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", container=\"dign-client\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Async\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545502804
Jul 17 03:12:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:56.786Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"shm\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/2c31834e99a6f2d3b1017c803937af69951091f07a82dc1c482cca7c4e85778b/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545505883
Jul 17 03:12:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:56.803Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_oom_events_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", container=\"traefik\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545509597
Jul 17 03:12:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:56.804Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_ports\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_endpoint=\"prometheus-node-exporter\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", port_name=\"metrics\", port_number=\"9100\", port_protocol=\"TCP\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545513389
Jul 17 03:12:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:56.843Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_limit_bytes\", asset=\"pcap-tsgx03\", asset_id=\"73\", container=\"configmap-reloader\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545511563
Jul 17 03:12:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:56.870Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_packets_dropped_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"cadvisor\", interface=\"eno1\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545508074
Jul 17 03:12:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:56.892Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-5e28b4cd\\\\x2d85df\\\\x2d47a2\\\\x2da9be\\\\x2daf9cd0223f13-volumes-kubernetes.io\\\\x7esecret-config.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689545503372
Jul 17 03:12:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:56.928Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-ens1f1np1.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689545503372
Jul 17 03:12:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:56.928Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_hwmon_sensor_label\", asset=\"pcap-tsgx03\", asset_id=\"73\", chip=\"pci0000:00_0000:00:19_3\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", label=\"tccd6\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", sensor=\"temp8\", service=\"prometheus-node-exporter\"}" timestamp=1689545505883
Jul 17 03:12:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:56.942Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_drop_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"br_dign_c\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545505883
Jul 17 03:12:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:56.974Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"23\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545505883
Jul 17 03:12:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:57.086Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_fifo_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"virtio_dign_s\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545505883
Jul 17 03:12:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:57.103Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_mod_info\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"github.com/prometheus/procfs\", nz_agent_id=\"3\", program=\"systemd_exporter\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", version=\"v0.0.11\"}" timestamp=1689545503372
Jul 17 03:12:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:57.119Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_drop_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens1f1np1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545505883
Jul 17 03:12:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:57.130Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"pcap-tsgx03\", asset_id=\"73\", condition=\"unknown\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"kube-multus-ds-tdbmj\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"7e892b02-c8d4-4893-b138-a1d4bb24f0fb\"}" timestamp=1689545513389
Jul 17 03:12:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:57.135Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_up_changes_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth8729f26e\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545505883
Jul 17 03:12:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:57.154Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_errs_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"vethd41b22b4\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545505883
Jul 17 03:12:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:57.172Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_down_changes_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"eno35np2\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545505883
Jul 17 03:12:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:57.184Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_multicast_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens3f0np0\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545505883
Jul 17 03:12:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:57.195Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-d50a87f26946e9f4abd93f805eab86f9fd66b0a6be641385c5ecce42d8632b63.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"scope\"}" timestamp=1689545503372
Jul 17 03:12:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:57.208Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-rfkill.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689545503372
Jul 17 03:12:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:57.284Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"pcap-tsgx03\", asset_id=\"73\", condition=\"false\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"dign-server-gvdww\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"8a18b7c9-5f34-4084-b472-fc3356545d52\"}" timestamp=1689545513389
Jul 17 03:12:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:57.340Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"tmpfs\", name=\"run-k3s-containerd-io.containerd.grpc.v1.cri-sandboxes-c7c3f554bb5564ecf3c0b65d130e40b6beaced6e6ad0c45c1efcc258a66f6217-shm.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689545503372
Jul 17 03:12:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:57.388Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_restart_policy\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-tsg-traffic-engine-numa-1-rc82w\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", type=\"OnFailure\", uid=\"35a456c8-6298-49a3-932c-3e0370b5d4f7\"}" timestamp=1689545513389
Jul 17 03:12:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:57.472Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_hwmon_sensor_label\", asset=\"pcap-tsgx03\", asset_id=\"73\", chip=\"pci0000:00_0000:00:19_3\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", label=\"tccd2\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", sensor=\"temp4\", service=\"prometheus-node-exporter\"}" timestamp=1689545505883
Jul 17 03:12:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:59.669Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-c6ffcf61\\\\x2d3641\\\\x2d43ac\\\\x2db76c\\\\x2db24b78ac0cd3-volume\\\\x2dsubpaths-config\\\\x2dpath-dign\\\\x2dserver\\\\x2ddns-1.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689545504879
Jul 17 03:12:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:59.745Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_namespace_labels\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"consul\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545507425
Jul 17 03:12:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:59.822Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"32\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545509083
Jul 17 03:12:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:59.838Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_multicast_packets_transmitted_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_0\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545509083
Jul 17 03:12:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:59.868Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_start_time\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-node-exporter-t6jf2\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"21204ec2-df19-472e-9ae1-06104ab28d13\"}" timestamp=1689545507425
Jul 17 03:12:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:59.886Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", state=\"running\"}" timestamp=1689545511225
Jul 17 03:12:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:59.912Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_spec_paused\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"tsg-traffic-engine-numa-0-firewall\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545507425
Jul 17 03:12:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:59.929Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_data_transmitted_bytes_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_7\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545509083
Jul 17 03:12:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:59.950Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"39\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545509083
Jul 17 03:12:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:59.956Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545509083
Jul 17 03:12:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:59.972Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_commands_duration_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cmd=\"info\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545505071
Jul 17 03:12:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:59.973Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"abrt-vmcore.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"service\"}" timestamp=1689545504879
Jul 17 03:12:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:12:59.992Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"tsg-clixon-backend.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"simple\", type=\"service\"}" timestamp=1689545504879
Jul 17 03:13:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:00.069Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_labels\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"ipmi-exporter\", uid=\"7a89ee1e-b964-48f2-b841-b8ae695ea5db\"}" timestamp=1689545507425
Jul 17 03:13:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:00.086Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"119\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545509083
Jul 17 03:13:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:00.180Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"108\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545509083
Jul 17 03:13:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:00.180Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_device_id\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"vethb1293876\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545509083
Jul 17 03:13:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:00.288Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_net_dev_group\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"eno1\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545509083
Jul 17 03:13:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:00.291Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_commands_failed_calls_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cmd=\"get\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545505071
Jul 17 03:13:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:00.312Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"45\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545509083
Jul 17 03:13:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:00.353Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"phy_rx_packets_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", dev=\"virtio_dign_c\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"mrzcpd-exporter\"}" timestamp=1689545510377
Jul 17 03:13:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:00.482Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db140\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545505071
Jul 17 03:13:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:00.523Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"auth-rpcgss-module.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"service\"}" timestamp=1689545504879
Jul 17 03:13:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:00.561Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_restarts_total\", asset=\"msh-tsgx01\", asset_id=\"51\", container=\"prometheus-pushgateway\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-pushgateway-bd476bcf9-gc4tb\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"3c8ac867-dce4-49c8-a599-46d660966b33\"}" timestamp=1689545507425
Jul 17 03:13:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:00.581Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2did-scsi\\\\x2d362cea7f085d4ec002c2b67bee386ff9d\\\\x2dpart1.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689545504879
Jul 17 03:13:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:00.876Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"nfs-idmapd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"service\"}" timestamp=1689545504879
Jul 17 03:13:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:00.936Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"44\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545509083
Jul 17 03:13:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:01.035Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_commands_duration_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cmd=\"zscore\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545505071
Jul 17 03:13:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:02.595Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_frame_total\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"eno35np2\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545518318
Jul 17 03:13:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:02.624Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2did-scsi\\\\x2dSDELL_PERC_H345_Adp_000c295d9e2f6d2b2c006d8284f0a7ce\\\\x2dpart3.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689545508447
Jul 17 03:13:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:02.950Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-2d6e25980861778bc311617104303c70a5b85392d07033e89537fecd7c806f53-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689545508447
Jul 17 03:13:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:02.998Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_heap_released_bytes\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545518318
Jul 17 03:13:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:03.070Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"msh-tsgx02\", asset_id=\"63\", condition=\"Available\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"tsg-traffic-engine-numa-0-firewall\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"unknown\"}" timestamp=1689545511412
Jul 17 03:13:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:03.333Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"auditd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"service\"}" timestamp=1689545508447
Jul 17 03:13:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:03.352Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_packets_total\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"eno1\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545518318
Jul 17 03:13:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:03.374Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2dpartlabel-TSG\\\\x2dOS\\\\x2dDATA.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689545508447
Jul 17 03:13:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:03.375Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_info\", asset=\"msh-tsgx02\", asset_id=\"63\", created_by_kind=\"Job\", created_by_name=\"helm-install-traefik\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", host_ip=\"10.10.20.162\", host_network=\"false\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", node=\"localhost\", nz_agent_id=\"3\", pod=\"helm-install-traefik-ph7rq\", pod_ip=\"10.42.0.136\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"eacc1d44-839a-43ee-bf5c-0bccf9558d70\"}" timestamp=1689545511412
Jul 17 03:13:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:03.853Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_heap_alloc_bytes\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", instance=\"10.10.20.159:10090\", type=\"global\"}" timestamp=1689545522989
Jul 17 03:13:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:03.867Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"prometheus_target_sync_length_seconds\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", instance=\"10.10.20.159:10090\", quantile=\"0.05\", scrape_job=\"261\", type=\"global\"}" timestamp=1689545522989
Jul 17 03:13:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:06.872Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-server-6ff784d758-76b74\", project=\"TSG-OS\", project_id=\"7\", reason=\"Evicted\", service=\"prometheus-kube-state-metrics\", uid=\"5ee8d190-8de7-4d53-a103-d0969342cef5\"}" timestamp=1689545521227
Jul 17 03:13:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:06.888Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"tmpfs\", name=\"var-lib-kubelet-pods-70fe949c\\\\x2dc79a\\\\x2d4c4d\\\\x2d9150\\\\x2d1de2e9791a04-volumes-kubernetes.io\\\\x7esecret-config.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689545513582
Jul 17 03:13:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:06.915Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-mqueue.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689545513582
Jul 17 03:13:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:06.978Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_protocol_type\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens6f1np1\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545523910
Jul 17 03:13:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:07.165Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db234\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545518290
Jul 17 03:13:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:07.519Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Tcp_Link_S2C\", app_name=\"sapp4\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", vsysId=\"1\"}" timestamp=1689545517817
Jul 17 03:13:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:07.894Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", phase=\"Unknown\", pod=\"dign-server-znxnp\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"7b4d8362-f626-49d3-b132-7bb684fd42a1\"}" timestamp=1689545521227
Jul 17 03:13:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:08.104Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_ready\", asset=\"twa-tsgx01\", asset_id=\"91\", container=\"kube-state-metrics\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-kube-state-metrics-6b768c7bf8-j4flh\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"d9f40692-7fb0-4b4c-ba8a-dc6b24fab188\"}" timestamp=1689545521227
Jul 17 03:13:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:08.110Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Pending\", pod=\"helm-install-tsg-traffic-engine-numa-0-8qr7q\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"fd6761e6-1e9e-4776-b037-dcda2dfd3380\"}" timestamp=1689545521227
Jul 17 03:13:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:08.120Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_ready\", asset=\"twa-tsgx01\", asset_id=\"91\", container=\"telegraf-security\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", pod=\"tsg-traffic-engine-numa-1-firewall-7f6cb768db-2lm6c\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"9b7a7ee8-3d4c-4d13-b2bb-d0637b89d0c8\"}" timestamp=1689545521227
Jul 17 03:13:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:08.129Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"18\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545523910
Jul 17 03:13:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:08.135Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_address_assign_type\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"cnibr0\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545523910
Jul 17 03:13:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:08.186Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cpu=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545523910
Jul 17 03:13:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:08.256Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_spec_completions\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", job_name=\"helm-install-prometheus\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545521227
Jul 17 03:13:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:08.370Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_address_assign_type\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens4f2np2\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545523910
Jul 17 03:13:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:08.407Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cpu=\"71\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545523910
Jul 17 03:13:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:08.427Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_cfs_throttled_periods_total\", asset=\"twa-tsgx01\", asset_id=\"91\", container=\"kube-multus\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545508871
Jul 17 03:13:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:08.514Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_writes_total\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689545514971
Jul 17 03:13:09 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:09.634Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_FlushOutputSize_99_9th_percentile\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Server\"}" timestamp=1689545528804
Jul 17 03:13:09 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:09.779Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_storeFileSize\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"troubleshooting_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"d2114d26e64525f0229a1f51892d452d\", table=\"index_partfile_troubleshooting_hos_bucket\", type=\"index\"}" timestamp=1689545528804
Jul 17 03:13:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:10.348Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_GcCountConcurrentMarkSweep\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", module=\"DC-HBase\", module_id=\"3\", name=\"JvmMetrics\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545528804
Jul 17 03:13:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:10.436Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_flushesQueuedCount\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"rtp_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"d64e085bec7dcaecff2f6350cdf00ff6\", table=\"index_time_rtp_hos_bucket\", type=\"index\"}" timestamp=1689545528804
Jul 17 03:13:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:10.473Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_numReferenceFiles\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"knowledge_base_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"2195076ef2d2041d654651ace4a5a2ab\", table=\"index_partfile_knowledge_base_hos_bucket\", type=\"index\"}" timestamp=1689545528804
Jul 17 03:13:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:10.881Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_replicaid\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"troubleshooting_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"f85e6fd815be42ad7194e26e839a5014\", table=\"index_partfile_troubleshooting_hos_bucket\", type=\"index\"}" timestamp=1689545528804
Jul 17 03:13:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:10.900Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_readRequestCount\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"assessment_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"4277b377c6aa7f26d6a7ca4321a94b79\", table=\"index_partfile_assessment_hos_bucket\", type=\"index\"}" timestamp=1689545528804
Jul 17 03:13:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:10.911Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_getCount\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"knowledge_base_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"fc3469e3d6e565b4e171251d5a25e11d\", table=\"index_filename_knowledge_base_hos_bucket\", type=\"index\"}" timestamp=1689545528804
Jul 17 03:13:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:10.928Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_compactionsCompletedCount\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", module=\"DC-HBase\", module_id=\"3\", namespace=\"SYSTEM\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"530046d786fd221ec50154f22c115fa4\", table=\"STATS\", type=\"data\"}" timestamp=1689545528804
Jul 17 03:13:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:11.153Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_appendCount\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"default\", project=\"OLAP\", project_id=\"3\", region=\"be2761dee59377fcbdeb66fdaae9c491\", table=\"index_time_default\", type=\"index\"}" timestamp=1689545528804
Jul 17 03:13:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:11.173Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_flushesQueuedCount\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"4cc00150a1946724d37ce029b51ab02f\", table=\"rtp_hos_bucket\", type=\"data\"}" timestamp=1689545528804
Jul 17 03:13:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:11.394Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_tsg_galaxy_table_gtpc_knowledge_base_metric_scanTime_99_9th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545530148
Jul 17 03:13:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:11.414Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_exceptions_multiResponseTooLarge\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"IPC\"}" timestamp=1689545530148
Jul 17 03:13:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:11.518Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_IndexPrepareTime_min\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"PhoenixIndexer\"}" timestamp=1689545530148
Jul 17 03:13:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:11.603Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_193_1_metric_majorCompactionInputFileCount_max\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545530148
Jul 17 03:13:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:11.724Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_MUTEX_metric_tableReadQueryPerSecond_count\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545530148
Jul 17 03:13:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:11.773Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_scanCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", namespace=\"SYSTEM\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"ccc86592096566c6926dda82e28a082a\", table=\"LOG\", type=\"data\"}" timestamp=1689545530148
Jul 17 03:13:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:11.775Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_194_2_metric_flushTime_99_9th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545530148
Jul 17 03:13:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:11.795Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_193_1_metric_flushOutputSize_min\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545530148
Jul 17 03:13:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:11.796Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_default_metric_incrementTime_mean\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545530148
Jul 17 03:13:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:11.908Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_3_metric_deleteBatchTime_max\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545530148
Jul 17 03:13:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:11.938Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_2_metric_compactionInputSize_median\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545530148
Jul 17 03:13:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:11.981Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_maxFlushQueueSize\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"58414b9635e704bfdbebc3459bee598f\", table=\"assessment_hos_bucket\", type=\"data\"}" timestamp=1689545530148
Jul 17 03:13:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:12.061Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_compactionsQueuedCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"assessment_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"9bf646028843a7e0768f61bf656cfb04\", table=\"index_time_assessment_hos_bucket\", type=\"index\"}" timestamp=1689545530148
Jul 17 03:13:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:12.073Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_MajorCompactionOutputSize_max\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Server\"}" timestamp=1689545530148
Jul 17 03:13:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:12.085Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_LOG_metric_incrementTime_num_ops\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545530148
Jul 17 03:13:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:12.100Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_numReferenceFiles\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"knowledge_base_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"13a4e6025bf464f6e219ed4362f86f50\", table=\"index_time_knowledge_base_hos_bucket\", type=\"index\"}" timestamp=1689545530148
Jul 17 03:13:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:12.112Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_readRequestCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"knowledge_base_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"408298ac4baa4d15f68738efc36b7451\", table=\"index_filename_knowledge_base_hos_bucket\", type=\"index\"}" timestamp=1689545530148
Jul 17 03:13:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:12.179Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_ScanTime_75th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Server\"}" timestamp=1689545530148
Jul 17 03:13:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:12.197Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_tsg_galaxy_table_gtpc_knowledge_base_metric_putTime_75th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545530148
Jul 17 03:13:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:12.240Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_194_1_metric_majorCompactionOutputFileCount_95th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545530148
Jul 17 03:13:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:12.493Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_storeFileSize\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"default\", project=\"OLAP\", project_id=\"3\", region=\"699dd1a09b72a827f105f7fd6cafcd02\", table=\"index_time_default\", type=\"index\"}" timestamp=1689545530148
Jul 17 03:13:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:12.638Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_TASK_metric_putBatchTime_min\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545530148
Jul 17 03:13:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:12.687Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_readRequestCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"knowledge_base_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"bf2a63dbae3b7c03de972a5d882a0894\", table=\"index_filename_knowledge_base_hos_bucket\", type=\"index\"}" timestamp=1689545530148
Jul 17 03:13:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:12.722Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_193_1_metric_splitTime_mean\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545530148
Jul 17 03:13:13 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:13.059Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_193_1_metric_compactionInputFileCount_75th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545530148
Jul 17 03:13:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:15.278Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db170\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545526027
Jul 17 03:13:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:15.307Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"49\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545521230
Jul 17 03:13:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:15.408Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"twa-tsgx07\", asset_id=\"103\", container=\"kube-state-metrics\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"cadvisor\", failure_type=\"pgfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"container\"}" timestamp=1689545524648
Jul 17 03:13:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:15.429Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.grpc.v1.cri-sandboxes-2c1282694572bf22945539738d3e76ea74b899f7d6d046e083c67d2aab230f0f-shm.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689545527046
Jul 17 03:13:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:15.453Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-bfab5aec2f5ac14ab66426bea0ced7ef31af14dd9d2883bdc43a220e1db53115-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689545527046
Jul 17 03:13:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:15.479Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"dign-client-cg2sp\", project=\"TSG-OS\", project_id=\"7\", reason=\"NodeLost\", service=\"prometheus-kube-state-metrics\", uid=\"f68561d2-9434-4bd0-905f-4bbeef0b0682\"}" timestamp=1689545528076
Jul 17 03:13:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:15.505Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_metadata_resource_version\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.tsg-traffic-engine-numa-0.v5\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545528076
Jul 17 03:13:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:15.672Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_reservation_limit_bytes\", asset=\"twa-tsgx07\", asset_id=\"103\", container=\"lb-tcp-80\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545529150
Jul 17 03:13:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:15.695Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"30\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545521230
Jul 17 03:13:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:15.714Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_info\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_endpoint=\"ipmi-exporter\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545528076
Jul 17 03:13:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:15.721Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-e42eb8ff72c7bf7a523addcbaba596e9042ab335d1b8ceb10d9bc8caebf953a8.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"scope\"}" timestamp=1689545527046
Jul 17 03:13:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:15.744Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"traefik-64f55bb67d-lzgmj\", project=\"TSG-OS\", project_id=\"7\", reason=\"UnexpectedAdmissionError\", service=\"prometheus-kube-state-metrics\", uid=\"18136b1d-dee5-4f2e-a352-07c6fe3b2565\"}" timestamp=1689545528076
Jul 17 03:13:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:15.779Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"125\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545521230
Jul 17 03:13:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:16.554Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_oom_events_total\", asset=\"twa-tsgx07\", asset_id=\"103\", container=\"redis\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545520066
Jul 17 03:13:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:16.682Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_info\", asset=\"twa-tsgx07\", asset_id=\"103\", container=\"exporter\", container_id=\"containerd://ec1efd8767d69ab4836a22c205a5a4e3c9504a5036c73dbe1f50bf73d625fb4d\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", image=\"quay.io/rebuy/exporter-merger:v0.2.0\", image_id=\"sha256:70980fec77625953a49ebfad047fdf278654100cecf3ac4ff77794143c97cfd4\", image_spec=\"quay.io/rebuy/exporter-merger:v0.2.0\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"systemd-exporter-7f549d9485-jkp9n\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"c435f080-d1a5-46a3-868f-b5d8209ce2ea\"}" timestamp=1689545528076
Jul 17 03:13:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:16.792Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_labels\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"cm-policy-local-cache-redis-master-69967b5879-p4gsc\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"d6907218-5827-4019-84f7-b49fadb75836\"}" timestamp=1689545528076
Jul 17 03:13:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:16.932Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_created\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.tsg-traffic-engine-numa-0.v5\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545528076
Jul 17 03:13:17 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:17.039Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-6adadf551ce1dda054cd6812f4f516a16930694b2bfd6009c53c19ada91f008b.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"scope\"}" timestamp=1689545527046
Jul 17 03:13:17 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:17.042Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"twa-tsgx07\", asset_id=\"103\", condition=\"true\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", pod=\"tsg-traffic-engine-numa-0-firewall-5b47889f4f-8qqtj\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"64082a69-3be2-4b5e-9e3a-c0da16576d9e\"}" timestamp=1689545528076
Jul 17 03:13:17 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:17.088Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"20\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545521230
Jul 17 03:13:17 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:17.170Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_device_id\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens8f1np1\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545521230
Jul 17 03:13:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:19.598Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_labels\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-0-firewall-645557869b\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545533949
Jul 17 03:13:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:19.638Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_oom_events_total\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689545532227
Jul 17 03:13:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:19.675Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_replicas\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-0-firewall-645557869b\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545533949
Jul 17 03:13:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:19.718Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_info\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_endpoint=\"traefik\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545533949
Jul 17 03:13:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:19.783Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"twa-tsgx06\", asset_id=\"101\", condition=\"Progressing\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"ipmi-exporter\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"true\"}" timestamp=1689545533949
Jul 17 03:13:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:19.812Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_mem_fragmentation_ratio\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545531998
Jul 17 03:13:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:19.840Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Udp_Bypass_Pkt\", app_name=\"sapp4\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", vsysId=\"1\"}" timestamp=1689545533544
Jul 17 03:13:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:19.919Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"scrape_series_added\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_instance=\"10.42.0.75:9121\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545531998
Jul 17 03:13:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:19.983Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"55\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545537612
Jul 17 03:13:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:20.010Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-module-configfs.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689545524356
Jul 17 03:13:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:20.073Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_created\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", job_name=\"helm-install-tsg-traffic-engine-numa-0\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545533949
Jul 17 03:13:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:20.104Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_owner\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", owner_is_controller=\"true\", owner_kind=\"Deployment\", owner_name=\"tsg-traffic-engine-numa-0-firewall\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-0-firewall-7dc8b57dfd\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545533949
Jul 17 03:13:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:20.131Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_up_changes_total\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens4f2np2\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545537612
Jul 17 03:13:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:20.551Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_waiting\", asset=\"pcap-tsgx10\", asset_id=\"89\", container=\"metrics\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"cm-policy-local-cache-redis-master-69967b5879-8s2nz\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"efdb7e93-9e91-4d51-b008-336f3a7336fb\"}" timestamp=1689545533828
Jul 17 03:13:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:20.558Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_hwmon_chip_names\", asset=\"twa-tsgx06\", asset_id=\"101\", chip=\"0000:80:01_2_0000:82:00_1\", chip_name=\"bnxt_en\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545537612
Jul 17 03:13:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:20.576Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_cpu_seconds_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-journald.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689545530657
Jul 17 03:13:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:20.597Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"tmpfs\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/var/lib/kubelet/pods/c7f82753-6cf1-4af2-b179-5ea64140eb6d/volumes/kubernetes.io~projected/kube-api-access-kt6tc\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545537612
Jul 17 03:13:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:20.618Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_drop_total\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens1f0np0\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545537612
Jul 17 03:13:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:20.684Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_created\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-alertmanager\", uid=\"fd83ad51-26c9-4768-abab-ece5a7a95bb2\"}" timestamp=1689545533949
Jul 17 03:13:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:20.700Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"56\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545537612
Jul 17 03:13:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:20.764Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_writes_total\", asset=\"twa-tsgx06\", asset_id=\"101\", container=\"kube-multus\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545518437
Jul 17 03:13:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:20.813Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"3\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545528263
Jul 17 03:13:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:20.828Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-9145d365\\\\x2d24c7\\\\x2d4ff1\\\\x2da1ee\\\\x2d0a800b33502b-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2ds857g.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689545524356
Jul 17 03:13:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:20.854Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"0\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545528263
Jul 17 03:13:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:21.092Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_reads_bytes_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689545522425
Jul 17 03:13:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:21.105Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"54\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545528263
Jul 17 03:13:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:21.149Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-9435b5376c50e459083851c857cbb78d7341f65d486c83f46d4c9810150bb88b.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"scope\"}" timestamp=1689545530657
Jul 17 03:13:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:21.177Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db50\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545534840
Jul 17 03:13:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:21.215Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"62\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545528263
Jul 17 03:13:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:21.240Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_bytes_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"cadvisor\", interface=\"eth0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689545532080
Jul 17 03:13:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:21.276Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx06\", asset_id=\"101\", container=\"dign-server-dns\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Sync\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545523195
Jul 17 03:13:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:21.491Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_mtu_bytes\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth96cee52c\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545528263
Jul 17 03:13:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:22.315Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_memory_AnonHugePages_bytes\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"OLAP-Node-Exporter-twa-dellr740-dt001\", endpoint_id=\"257\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545541405
Jul 17 03:13:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:22.838Z caller=dedupe.go:112 component=remote level=error remote_name=52db6f url=http://10.10.10.159:10090/nz-agent/prometheus/proxy/api/v1/write msg="non-recoverable error" count=1000 exemplarCount=0 err="server returned HTTP status 400 : out of order sample"
Jul 17 03:13:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:23.750Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"msh-tsgx09\", asset_id=\"55\", condition=\"unknown\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-delete-tsg-traffic-engine-numa-1-bjvt6\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"55e15df4-2d3f-4878-94dc-5fef3e71fbdb\"}" timestamp=1689545540068
Jul 17 03:13:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:23.791Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-a23be08d03472f232c70636e66be0d6459ca24fa4775e96bbb9c875cd678b726-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689545538416
Jul 17 03:13:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:24.233Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx09\", asset_id=\"55\", cpu=\"93\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545534790
Jul 17 03:13:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:24.264Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db38\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545537019
Jul 17 03:13:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:24.288Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db24\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545537019
Jul 17 03:13:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:24.375Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_restart_policy\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", pod=\"kubernetes-dashboard-66dd5b456-rs7kq\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", type=\"Always\", uid=\"cbd312de-6144-4424-8813-fba2b3b16e9b\"}" timestamp=1689545531122
Jul 17 03:13:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:24.530Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_created\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"mrzcpd-exporter\", uid=\"dfc53202-8aee-46de-83cb-304b87e00648\"}" timestamp=1689545531122
Jul 17 03:13:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:24.554Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db160\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545541795
Jul 17 03:13:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:24.724Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_protocol_type\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"br_dign_c\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545540160
Jul 17 03:13:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:24.781Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_sector_writes_total\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/run\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545530300
Jul 17 03:13:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:24.785Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_device_id\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens4f0np0\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545540160
Jul 17 03:13:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:24.798Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_latest_fork_seconds\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545541795
Jul 17 03:13:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:24.813Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_annotations\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", job_name=\"helm-delete-tsg-traffic-engine-numa-1\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545540068
Jul 17 03:13:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:24.817Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-bc5fa7cd79f33e11f1d62cbfd0be5ad4c95b759e2dfc75ab24dbdd03f54271ec.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"scope\"}" timestamp=1689545531102
Jul 17 03:13:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:24.830Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx09\", asset_id=\"55\", cpu=\"125\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545534790
Jul 17 03:13:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:26.070Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"pcap-kvm-cm001\", asset_id=\"45\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db63\", endpoint=\"CM-Branch-Redis-pcap-kvm-cm001\", endpoint_id=\"321\", host=\"192.168.10.157\", module=\"CM-Branch-Redis\", module_id=\"73\", nz_agent_id=\"3\", parent_asset=\"pcap-dellr740-pve001\", parent_asset_id=\"43\", project=\"Central-Management\", project_id=\"5\"}" timestamp=1689545545365
Jul 17 03:13:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:26.282Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"nsfs\", name=\"run-netns-cni\\\\x2d800585cd\\\\x2daed5\\\\x2dcc33\\\\x2d4a7b\\\\x2de391b804706b.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689545533907
Jul 17 03:13:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:26.329Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"twa-tsgx09\", asset_id=\"107\", container=\"kube-state-metrics\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"stopped\"}" timestamp=1689545541379
Jul 17 03:13:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:26.332Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Succeeded\", pod=\"helm-install-traefik-vw4k2\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"f01b2d19-81c4-4465-a1a9-08cbc2270574\"}" timestamp=1689545538744
Jul 17 03:13:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:26.382Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"8\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545535810
Jul 17 03:13:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:26.419Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"13\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545535810
Jul 17 03:13:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:26.438Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"81\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545535810
Jul 17 03:13:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:26.457Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"74\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545535810
Jul 17 03:13:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:26.482Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"30\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545535810
Jul 17 03:13:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:26.517Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"30\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545535810
Jul 17 03:13:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:26.606Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_files_free\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"shm\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/afefe92ea84fa41dbbadb1687ce5c63d370abc6861879fe40c18cca568cc3018/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545535810
Jul 17 03:13:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:26.728Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_compressed_total\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vetha8a8b4ed\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545535810
Jul 17 03:13:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:26.752Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"41\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545535810
Jul 17 03:13:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:27.319Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link_mode\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth188af860\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545535810
Jul 17 03:13:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:27.358Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-binfmt.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"oneshot\", type=\"service\"}" timestamp=1689545533907
Jul 17 03:13:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:27.466Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_annotations\", asset=\"twa-tsgx09\", asset_id=\"107\", configmap=\"cluster-dns\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545538744
Jul 17 03:13:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:27.480Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"11\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545535810
Jul 17 03:13:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:27.506Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_colls_total\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth255034d1\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545535810
Jul 17 03:13:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:27.804Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"unbound-anchor.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689545534114
Jul 17 03:13:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:27.820Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2dpath-pci\\\\x2d0000:81:00.0\\\\x2dscsi\\\\x2d0:2:0:0\\\\x2dpart2.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689545534114
Jul 17 03:13:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:27.939Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_usage_bytes\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"prometheus-alertmanager\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545537244
Jul 17 03:13:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:27.991Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"tsg-clixon-backend.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689545534114
Jul 17 03:13:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:28.107Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Failed\", pod=\"helm-install-traefik-crd-m2nsh\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"6ee7b6d8-27b2-4118-9cf0-9229465d0b5b\"}" timestamp=1689545535682
Jul 17 03:13:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:28.162Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_alloc_bytes_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545539446
Jul 17 03:13:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:28.295Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_labels\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"kubernetes-dashboard-66dd5b456\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545535682
Jul 17 03:13:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:28.338Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_info\", asset=\"msh-tsgx10\", asset_id=\"69\", created_by_kind=\"DaemonSet\", created_by_name=\"dign-server\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", host_ip=\"10.10.20.170\", host_network=\"false\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", node=\"localhost\", nz_agent_id=\"3\", pod=\"dign-server-pf96k\", pod_ip=\"10.42.0.166\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"75dc4160-da0a-4767-823a-227b8dd303c1\"}" timestamp=1689545535682
Jul 17 03:13:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:28.401Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_size_bytes\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda2\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", fstype=\"ext4\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/mnt/onie-boot\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545542882
Jul 17 03:13:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:28.433Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_symbol_error_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_2\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545542882
Jul 17 03:13:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:28.456Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_virtual_memory_bytes\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"gssproxy.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689545534114
Jul 17 03:13:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:28.493Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_device_id\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"vethbd530c93\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545542882
Jul 17 03:13:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:28.583Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_created\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"chart-values-configmap-reloader\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545535682
Jul 17 03:13:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:28.660Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_commands_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cmd=\"set\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545539446
Jul 17 03:13:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:28.746Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_compressed_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"vethd626638d\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545542882
Jul 17 03:13:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:28.752Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sysstat-summary.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"oneshot\", type=\"service\"}" timestamp=1689545534114
Jul 17 03:13:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:31.411Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_compressed_total\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"vethc8e6a12a\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545540939
Jul 17 03:13:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:31.445Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_writes_bytes_total\", asset=\"msh-tsgx05\", asset_id=\"67\", container=\"prometheus-server-configmap-reload\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545528019
Jul 17 03:13:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:31.460Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_packets_dropped_total\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"cadvisor\", interface=\"ens8f1np1\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545533645
Jul 17 03:13:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:31.517Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_virtual_memory_max_bytes\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"tsg-clixon-restconf.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689545541015
Jul 17 03:13:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:31.541Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_gc_duration_seconds\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", quantile=\"0.75\", service=\"prometheus-node-exporter\"}" timestamp=1689545540939
Jul 17 03:13:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:31.553Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", pod=\"tsg-traffic-engine-numa-0-firewall-76b5c5788d-2cs6q\", project=\"TSG-OS\", project_id=\"7\", reason=\"Shutdown\", service=\"prometheus-kube-state-metrics\", uid=\"99e7b7f0-307f-4274-a670-faf429f178f7\"}" timestamp=1689545544068
Jul 17 03:13:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:31.721Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"98\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545540939
Jul 17 03:13:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:31.734Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_errs_total\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens1f0np0\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545539093
Jul 17 03:13:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:31.737Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"22\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545539093
Jul 17 03:13:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:31.764Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_packets_total\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens1f0np0\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545539093
Jul 17 03:13:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:31.800Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"twa-tsgx03\", asset_id=\"95\", container=\"exporter\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"iowaiting\"}" timestamp=1689545532043
Jul 17 03:13:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:31.817Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_changes_total\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth171e80b1\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545539093
Jul 17 03:13:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:31.861Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"microcode.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689545546907
Jul 17 03:13:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:31.875Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"telegraf\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", state=\"sleeping\"}" timestamp=1689545535805
Jul 17 03:13:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:31.881Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_commands_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cmd=\"zadd\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545546438
Jul 17 03:13:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:31.893Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas_unavailable\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"mrzcpd-exporter\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545539276
Jul 17 03:13:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:31.901Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"43\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545540791
Jul 17 03:13:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:31.932Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_ready_replicas\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"kubernetes-dashboard-66dd5b456\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545539276
Jul 17 03:13:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:32.060Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.grpc.v1.cri-sandboxes-4b859679167266693e598bdb856abed42c9d9c54368729532d2b0d941022ca52-shm.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689545541015
Jul 17 03:13:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:32.112Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"83\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545540791
Jul 17 03:13:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:32.147Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:20-0000:20:03.1-0000:21:00.0-infiniband_mad-umad2.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689545542644
Jul 17 03:13:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:32.428Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-virtual-net-veth15723f66.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689545542644
Jul 17 03:13:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:32.460Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545539093
Jul 17 03:13:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:32.525Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_created\", asset=\"twa-tsgx03\", asset_id=\"95\", configmap=\"sapp-tsg-traffic-engine-numa-1\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545550436
Jul 17 03:13:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:32.555Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db119\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545536711
Jul 17 03:13:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:32.638Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-932a67aaa5bdeadf61c2470d22336aefc62897c6245af91f57e5bbb294dba553.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"scope\"}" timestamp=1689545542644
Jul 17 03:13:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:32.673Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_created\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"shaping-announce-port-tsg-traffic-engine-numa-1\", uid=\"22522065-ec14-40f7-8e23-6ae1a30cbcfd\"}" timestamp=1689545550436
Jul 17 03:13:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:32.728Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled_time\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", pod=\"kubernetes-dashboard-66dd5b456-nqg8s\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"e02c929a-3c07-4477-b04a-8ffc16c405f5\"}" timestamp=1689545539894
Jul 17 03:13:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:32.735Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"22\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545540791
Jul 17 03:13:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:32.839Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_read_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/run\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545541927
Jul 17 03:13:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:32.864Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"30\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545539093
Jul 17 03:13:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:32.921Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-7ceecbf934242885767971d44d70ba11e527e62b91a840a259a89624e3450cd0-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689545542644
Jul 17 03:13:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:32.970Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_threads_max\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689545538665
Jul 17 03:13:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:32.974Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_drop_total\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethf7cbe37a\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545539093
Jul 17 03:13:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:32.993Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"systemd-exporter-7f549d9485-bph7s\", project=\"TSG-OS\", project_id=\"7\", reason=\"UnexpectedAdmissionError\", service=\"prometheus-kube-state-metrics\", uid=\"758d8a04-3fb4-4af1-97bd-b2b3e1439f97\"}" timestamp=1689545539276
Jul 17 03:13:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:33.006Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"6\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545539093
Jul 17 03:13:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:33.079Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_colls_total\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens2f1np1\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545540791
Jul 17 03:13:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:33.241Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"twa-tsgx05\", asset_id=\"99\", container=\"telegraf\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", state=\"iowaiting\"}" timestamp=1689545535566
Jul 17 03:13:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:33.260Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-ttyS3.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689545546907
Jul 17 03:13:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:33.260Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_errs_total\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethc2cbb116\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545540605
Jul 17 03:13:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:33.264Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_queue_length\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens4f2np2\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545540791
Jul 17 03:13:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:33.314Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db121\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545549260
Jul 17 03:13:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:33.356Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-6a692a6c37775cfac2c8d862aac1e1c484c074fd3c884467f03541ec8697bb21.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"scope\"}" timestamp=1689545547691
Jul 17 03:13:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:33.362Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2dpartlabel-EFI\\\\x5cx20System.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689545547691
Jul 17 03:13:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:33.372Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-pod9982afd0_5736_4dce_b764_2f7b0c30831a.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"slice\"}" timestamp=1689545547691
Jul 17 03:13:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:33.379Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-netns-cni\\\\x2d166b1a3d\\\\x2d4b1a\\\\x2d1980\\\\x2d7bf4\\\\x2dd6add3c72f10.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689545547691
Jul 17 03:13:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:33.475Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"16\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545540791
Jul 17 03:13:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:33.523Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_created\", asset=\"twa-tsgx03\", asset_id=\"95\", configmap=\"tfe-tsg-traffic-engine-numa-1\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545550436
Jul 17 03:13:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:33.597Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db241\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545536711
Jul 17 03:13:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:33.621Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_complete\", asset=\"twa-tsgx05\", asset_id=\"99\", condition=\"unknown\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", job_name=\"helm-install-traefik-crd\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545539894
Jul 17 03:13:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:33.642Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_created\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", pod=\"kubernetes-dashboard-66dd5b456-nqg8s\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"e02c929a-3c07-4477-b04a-8ffc16c405f5\"}" timestamp=1689545539894
Jul 17 03:13:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:33.687Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"114\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545540791
Jul 17 03:13:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:33.880Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"initrd-cleanup.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"service\"}" timestamp=1689545547691
Jul 17 03:13:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:33.905Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_errs_total\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens6f0np0\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545540605
Jul 17 03:13:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:33.980Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"nfs-server.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"service\"}" timestamp=1689545547691
Jul 17 03:13:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:34.081Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_ready\", asset=\"twa-tsgx05\", asset_id=\"99\", container=\"firewall\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", pod=\"tsg-traffic-engine-numa-1-firewall-6dd49d747-slxl4\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"bdd10b0d-cc33-4a4f-9622-89ebf267e6c3\"}" timestamp=1689545539894
Jul 17 03:13:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:34.204Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens1f0np0\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545540605
Jul 17 03:13:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:34.324Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_start_time\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-mrzcpd-exporter-dmnbs\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"1982d9e9-ffea-4312-9282-d3a102867809\"}" timestamp=1689545539894
Jul 17 03:13:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:34.380Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"twa-tsgx05\", asset_id=\"99\", condition=\"false\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"kube-multus-ds-vmk6p\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"e3971f33-46bb-450e-a67b-8e9e2f2ec265\"}" timestamp=1689545539894
Jul 17 03:13:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:34.402Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"28\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545540605
Jul 17 03:13:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:34.524Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db73\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545549260
Jul 17 03:13:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:34.533Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"svclb-traefik-f8ab0b79-f6t4m\", project=\"TSG-OS\", project_id=\"7\", reason=\"Shutdown\", service=\"prometheus-kube-state-metrics\", uid=\"6c839a6b-757d-460b-acb3-20a2f3df359e\"}" timestamp=1689545539894
Jul 17 03:13:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:34.552Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_owner\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", owner_is_controller=\"true\", owner_kind=\"Deployment\", owner_name=\"tsg-traffic-engine-numa-0-firewall\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-0-firewall-7cbb5bf99c\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545539894
Jul 17 03:13:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:40.519Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"127\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545551647
Jul 17 03:13:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:40.531Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"35\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545551647
Jul 17 03:13:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:40.751Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-d4cded8dd7205c4abd00951d49c4e36eeed11de68212005940e42ee4b259362a-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689545555764
Jul 17 03:13:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:40.790Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_down_changes_total\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens7f1np1\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545551647
Jul 17 03:13:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:40.936Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_info\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"chart-values-traefik\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545560034
Jul 17 03:13:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:41.050Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_filteredReadRequestCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"rtp_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"59ccbcf963afe69ff51b6e73fda0c71c\", table=\"index_time_rtp_hos_bucket\", type=\"index\"}" timestamp=1689545559757
Jul 17 03:13:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:41.073Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db111\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545553827
Jul 17 03:13:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:41.099Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-ed0bc7a6021e6bfc58af02d604b19850e1c310869c4fe29a29338829c2479e6f-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689545555764
Jul 17 03:13:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:41.438Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_reads_bytes_total\", asset=\"twa-tsgx04\", asset_id=\"97\", container=\"kubernetes-dashboard\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545538376
Jul 17 03:13:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:41.585Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_fully_labeled_replicas\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"systemd-exporter-7f549d9485\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545560034
Jul 17 03:13:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:41.681Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_numFilesCompactedCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", namespace=\"tsg_galaxy\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"7fed3599680b8cd8d453ece7a92c43c0\", table=\"gtpc_knowledge_base\", type=\"data\"}" timestamp=1689545559757
Jul 17 03:13:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:41.845Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"58\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545551647
Jul 17 03:13:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:42.604Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_running\", asset=\"pcap-tsgx01\", asset_id=\"71\", container=\"ipmi-exporter\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"ipmi-exporter-6f8464dd94-sr5t7\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"9f0744bb-41c2-4a60-a05e-c390d7032c33\"}" timestamp=1689545555154
Jul 17 03:13:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:42.632Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", container=\"prometheus-node-exporter\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Read\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545551274
Jul 17 03:13:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:42.696Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_load_average_10s\", asset=\"pcap-tsgx01\", asset_id=\"71\", container=\"telegraf\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689545551549
Jul 17 03:13:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:42.861Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_init_container_info\", asset=\"pcap-tsgx01\", asset_id=\"71\", container=\"firewall-init\", container_id=\"containerd://a06eec19b573cca9a2ed6038ddd9c3272dfd7e3828140f556bc7adfe4174b830\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", image=\"registry.gdnt-cloud.website/tsg-init:v23.07-rc1-079e964\", image_id=\"sha256:c3a7da59ebe9d7390fc09b3b4d50c14674963d0d9616b631fe8029ed3bd0e7d1\", image_spec=\"registry.gdnt-cloud.website/tsg-init:v23.07-rc1-079e964\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", pod=\"tsg-traffic-engine-numa-0-firewall-849b856b46-97zks\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"d20d4034-164e-4450-a851-056ee84ef97e\"}" timestamp=1689545555154
Jul 17 03:13:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:42.883Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Failed\", pod=\"helm-install-prometheus-qzxgz\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"5bc779e9-f51a-4b57-87de-ce5b690c25f8\"}" timestamp=1689545555154
Jul 17 03:13:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:43.143Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_queue_length\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens4f1np1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545556242
Jul 17 03:13:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:43.163Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-sysusers.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"oneshot\", type=\"service\"}" timestamp=1689545560671
Jul 17 03:13:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:43.190Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-netns-cni\\\\x2d2a048e9b\\\\x2dbe89\\\\x2dcb6d\\\\x2d64a4\\\\x2daa422971f757.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689545560671
Jul 17 03:13:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:43.261Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_packets_total\", asset=\"msh-tsgx06\", asset_id=\"61\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth6556db13\", endpoint=\"TSG-OS-msh-tsgx06\", endpoint_id=\"429\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545547064
Jul 17 03:13:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:43.277Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"msh-tsgx06\", asset_id=\"61\", cpu=\"71\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx06\", endpoint_id=\"429\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545547064
Jul 17 03:13:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:43.289Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db245\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545554838
Jul 17 03:13:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:43.340Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-random-seed.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689545560671
Jul 17 03:13:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:43.559Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", cpu=\"34\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Gohangout-msh-dellr740-dt002\", endpoint_id=\"167\", mode=\"nice\", module=\"DC-Gohangout\", module_id=\"35\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545562623
Jul 17 03:13:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:43.672Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_packets_dropped_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"cadvisor\", interface=\"br_dign_c\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545557938
Jul 17 03:13:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:43.697Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_start_time_seconds\", asset=\"pcap-tsgx01\", asset_id=\"71\", container=\"exporter-merger\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545560951
Jul 17 03:13:44 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:44.034Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db168\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545554838
Jul 17 03:13:44 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:44.087Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"shm\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/d4f883aa32612f9d55b97bc538ad459ecb5083c2cc45decc830671053aca5e3b/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545556242
Jul 17 03:13:44 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:44.108Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_readonly\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda4\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", fstype=\"ext4\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545556242
Jul 17 03:13:44 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:44.123Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-3e9e67b41c0b1529725e4cf54fb5208203db94be7088a128a55953a2d7b96e5d.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"scope\"}" timestamp=1689545560671
Jul 17 03:13:44 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:44.325Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", cpu=\"127\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545556242
Jul 17 03:13:44 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:44.342Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", cpu=\"19\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545556242
Jul 17 03:13:44 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:44.345Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_usage_bytes\", asset=\"pcap-tsgx01\", asset_id=\"71\", container=\"redis\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545559250
Jul 17 03:13:44 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:44.358Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Udp_Bypass_Bit\", app_name=\"sapp4\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", vsysId=\"1\"}" timestamp=1689545547581
Jul 17 03:13:44 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:44.382Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_labels\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"tsg-traffic-engine-numa-1-firewall\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545555154
Jul 17 03:13:47 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:47.939Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_NameNode_MemHeapMaxM\", asset=\"msh-dellr740-olap004\", asset_id=\"17\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Hadoop-msh-dellr740-olap004-NameNode\", endpoint_id=\"325\", module=\"NC-Hadoop\", module_id=\"37\", name=\"JvmMetrics\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545566265
Jul 17 03:13:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:49.394Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Get_75th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Server\"}" timestamp=1689545567439
Jul 17 03:13:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:49.539Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas_available\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"ipmi-exporter\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545559668
Jul 17 03:13:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:49.613Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Running\", pod=\"helm-install-systemd-exporter-vq7tn\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"8760a158-2fa4-4332-99b5-4233fdf937ac\"}" timestamp=1689545559668
Jul 17 03:13:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:49.679Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_194_2_metric_splitTime_99_9th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545567439
Jul 17 03:13:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:49.713Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_labels\", asset=\"twa-tsgx10\", asset_id=\"109\", configmap=\"kube-apiserver-legacy-service-account-token-tracking\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545559668
Jul 17 03:13:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:49.726Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx10\", asset_id=\"109\", cpu=\"115\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545561797
Jul 17 03:13:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:49.761Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_storeCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"knowledge_base_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"d4e72b6e91a9b75ed312cff599c6a2fa\", table=\"index_partfile_knowledge_base_hos_bucket\", type=\"index\"}" timestamp=1689545567439
Jul 17 03:13:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:49.774Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.grpc.v1.cri-sandboxes-1288df725fcbfecd4fdbd8a0e3571cb004951dade8125880acc73bb9945c1a5a-shm.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689545561057
Jul 17 03:13:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:49.778Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-d241c46bf3ffe9e923cd45bd83474b25ea0144a3b7d3ef28fecd11c30dd38352.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"scope\"}" timestamp=1689545561057
Jul 17 03:13:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:49.786Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_4_metric_majorCompactionOutputFileCount_25th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545567439
Jul 17 03:13:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:49.794Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-0b9ea40b5cffd9ff69eecac490bb63436c5751e7768ab0bf0cad51498310c4f9-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689545561057
Jul 17 03:13:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:49.802Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_numBytesCompactedCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"4821cc6d0e06e36fa88a4933ecc65476\", table=\"proxy_hos_bucket\", type=\"data\"}" timestamp=1689545567439
Jul 17 03:13:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:49.811Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"time-sync.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"target\"}" timestamp=1689545561057
Jul 17 03:13:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:49.817Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_193_1_metric_getTime_max\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545567439
Jul 17 03:13:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:49.828Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"52\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545566709
Jul 17 03:13:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:49.868Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_up_changes_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth1e5d3348\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545566709
Jul 17 03:13:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:49.872Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_packets_total\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"br-fd05c54133ba\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-olap003\", endpoint_id=\"235\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"yarn_resourcemanager,zookeeper,nacos,kafka,spark,druid_data,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545569716
Jul 17 03:13:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:49.882Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_scaling_frequency_hertz\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"20\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545566709
Jul 17 03:13:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:49.893Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_speed_bytes\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethab95fd5d\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545566709
Jul 17 03:13:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:49.898Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_scaling_frequency_min_hertz\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545566709
Jul 17 03:13:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:49.907Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_labels\", asset=\"twa-tsgx10\", asset_id=\"109\", configmap=\"sapp-tsg-traffic-engine-numa-1\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545559668
Jul 17 03:13:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:49.909Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"0\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545566709
Jul 17 03:13:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:49.917Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_flushesQueuedCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"default\", project=\"OLAP\", project_id=\"3\", region=\"7bee94458a9198e9118a337171739067\", table=\"index_partfile_default\", type=\"index\"}" timestamp=1689545567439
Jul 17 03:13:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:49.929Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx10\", asset_id=\"109\", cpu=\"66\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545561797
Jul 17 03:13:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:49.931Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_compactionsCompletedCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"knowledge_base_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"d4e72b6e91a9b75ed312cff599c6a2fa\", table=\"index_partfile_knowledge_base_hos_bucket\", type=\"index\"}" timestamp=1689545567439
Jul 17 03:13:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:49.934Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_threads_max\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"dign-server-dns\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545555576
Jul 17 03:13:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:49.952Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_running\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"helm\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-tsg-traffic-engine-numa-0-7c4md\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"c79c6928-ce63-4bb0-b43a-0334cbcfb354\"}" timestamp=1689545559668
Jul 17 03:13:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:49.973Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_virtual_memory_bytes\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dbus.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689545561057
Jul 17 03:13:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:49.988Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-ba106c44\\\\x2dd5c3\\\\x2d446a\\\\x2db10b\\\\x2d530b2492f73f-volume\\\\x2dsubpaths-config\\\\x2dvolume-telegraf\\\\x2dsecurity-1.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689545561057
Jul 17 03:13:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:50.021Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"48\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545566709
Jul 17 03:13:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:50.035Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"var-lib-kubelet-pods-a2024985\\\\x2dd999\\\\x2d4ae2\\\\x2d847f\\\\x2da93fc2842f87-volumes-kubernetes.io\\\\x7esecret-cm\\\\x2dcache\\\\x2dextra\\\\x2dsecrets.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689545561057
Jul 17 03:13:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:50.063Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_packets_transmitted_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_0\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545566709
Jul 17 03:13:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:50.083Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:20-0000:20:03.1-0000:21:00.1-infiniband-mlx5_3.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689545561057
Jul 17 03:13:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:50.102Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_tasks_max\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-ask-password-plymouth.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689545561057
Jul 17 03:13:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:50.161Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-6e4f2f4687bae3bbca1a2a2cbbc5a63313dfd06a46f8ecb30e6b1173c86a1f59-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689545566554
Jul 17 03:13:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:50.227Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_compressed_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth9afe06eb\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545566709
Jul 17 03:13:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:50.269Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_numBytesCompactedCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"default\", project=\"OLAP\", project_id=\"3\", region=\"668ca3489d8c6fec92a10db8b81f6e41\", table=\"index_partfile_default\", type=\"index\"}" timestamp=1689545567439
Jul 17 03:13:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:50.396Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_errs_total\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethbd8ba9ba\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545561797
Jul 17 03:13:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:50.398Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_annotations\", asset=\"twa-tsgx10\", asset_id=\"109\", configmap=\"kube-root-ca.crt\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545559668
Jul 17 03:13:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:50.431Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_info\", asset=\"twa-tsgx10\", asset_id=\"109\", configmap=\"kube-root-ca.crt\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545559668
Jul 17 03:13:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:50.447Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_metadata_resource_version\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.systemd-exporter.v1\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545561568
Jul 17 03:13:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:50.467Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"100\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545566709
Jul 17 03:13:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:50.495Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", phase=\"Succeeded\", pod=\"prometheus-kube-state-metrics-6b768c7bf8-n4ck4\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"7be94d73-b8c6-4b83-adb9-10dac35bf41e\"}" timestamp=1689545559668
Jul 17 03:13:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:50.524Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"twa-tsgx10\", asset_id=\"109\", cpu=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545561797
Jul 17 03:13:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:50.565Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"87\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545566709
Jul 17 03:13:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:50.577Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_free_bytes\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"shm\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/a379a67869d57aedcb0526bd2cb5e1a6fe7faed6994dcda2a2fc2426d6838604/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545566709
Jul 17 03:13:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:50.657Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"twa-tsgx08\", asset_id=\"105\", condition=\"Available\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"metrics-server\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"unknown\"}" timestamp=1689545561568
Jul 17 03:13:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:50.704Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_up_changes_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth5b68c36f\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545566709
Jul 17 03:13:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:50.706Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_address_available\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_endpoint=\"ipmi-exporter\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545561568
Jul 17 03:13:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:50.827Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_frame_total\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethe417278f\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545561797
Jul 17 03:13:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:50.873Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db246\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545566625
Jul 17 03:13:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:50.894Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_terminated\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"ipmi-exporter\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"ipmi-exporter-6f8464dd94-82g97\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"b95f152c-994f-41b5-9701-bd977efd71db\"}" timestamp=1689545559668
Jul 17 03:13:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:50.938Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_info\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"kubernetes-dashboard-key-holder\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545559668
Jul 17 03:13:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:50.975Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"configmap-reloader\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Sync\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545557921
Jul 17 03:13:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:50.990Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_colls_total\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethe7bf01ed\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545561797
Jul 17 03:13:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:51.005Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_scaling_frequency_hertz\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"115\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545566709
Jul 17 03:13:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:51.085Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_oom_events_total\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"firewall\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689545553253
Jul 17 03:13:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:51.141Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"54\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545566709
Jul 17 03:13:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:51.189Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-b15fff941476cef70462b09f05f0ee585143f5494ed72a0eb76e187092634ecf-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689545561057
Jul 17 03:13:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:51.205Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_bytes_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens1f1np1\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545566709
Jul 17 03:13:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:51.222Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_queue_length\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth5b68c36f\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545566709
Jul 17 03:13:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:51.564Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_frequency_min_hertz\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"75\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545566709
Jul 17 03:13:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:52.470Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_drop_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"br_dign_s\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545569337
Jul 17 03:13:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:52.601Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db141\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545559367
Jul 17 03:13:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:52.626Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_annotations\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"dashboard-metrics-scraper\", uid=\"ff2ed267-62f8-4a22-9d53-7cb3afc27d29\"}" timestamp=1689545564478
Jul 17 03:13:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:52.652Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"40\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545569337
Jul 17 03:13:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:52.667Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db231\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545559367
Jul 17 03:13:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:52.686Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", cpu=\"5\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545569337
Jul 17 03:13:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:52.825Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_mtu_bytes\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth2169720f\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545569337
Jul 17 03:13:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:52.836Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_threads\", asset=\"pcap-dellr740-dt002\", asset_id=\"33\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"NC-Gohangout-pcap-dellr740-dt002\", endpoint_id=\"173\", module=\"DC-Gohangout\", module_id=\"35\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545572667
Jul 17 03:13:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:53.060Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-dellr740-dt002\", asset_id=\"33\", cpu=\"39\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"NC-Gohangout-pcap-dellr740-dt002\", endpoint_id=\"173\", mode=\"softirq\", module=\"DC-Gohangout\", module_id=\"35\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545572667
Jul 17 03:13:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:53.153Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dnf-makecache.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"oneshot\", type=\"service\"}" timestamp=1689545572135
Jul 17 03:13:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:53.198Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_sector_reads_total\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545550046
Jul 17 03:13:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:53.221Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_compressed_total\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"cnibr0\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545568328
Jul 17 03:13:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:53.241Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-rdma-devices-mlx5_3:1-umad.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689545572135
Jul 17 03:13:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:53.387Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx12\", asset_id=\"113\", cpu=\"112\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545563391
Jul 17 03:13:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:53.400Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"msh-tsgx12\", asset_id=\"113\", cpu=\"71\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545563391
Jul 17 03:13:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:53.424Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_fully_labeled_replicas\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"systemd-exporter-7f549d9485\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545568663
Jul 17 03:13:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:53.449Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_id\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth401fe2c4\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545563391
Jul 17 03:13:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:53.536Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"phy_tx_bits_total\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", dev=\"ens1f0np0\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"mrzcpd-exporter\"}" timestamp=1689545564559
Jul 17 03:13:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:53.559Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"msh-tsgx12\", asset_id=\"113\", cpu=\"33\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545563391
Jul 17 03:13:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:53.572Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db9\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545558659
Jul 17 03:13:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:53.572Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2dpartlabel-ONIE\\\\x2dBOOT.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689545571179
Jul 17 03:13:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:53.578Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_avail_bytes\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"shm\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/7512c673cf7ffa28dd5a904fa1484f8255f3c895be156a23d940681ab8b297a3/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545568328
Jul 17 03:13:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:53.681Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_tasks_current\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-udev-settle.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689545560884
Jul 17 03:13:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:53.684Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cpu=\"20\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545559409
Jul 17 03:13:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:53.699Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-initctl.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"service\"}" timestamp=1689545571179
Jul 17 03:13:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:53.741Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-a188b741\\\\x2d4451\\\\x2d4393\\\\x2da8d5\\\\x2d35ceaab41710-volume\\\\x2dsubpaths-config\\\\x2dvolume-telegraf-2.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689545571179
Jul 17 03:13:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:53.743Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", cpu=\"40\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545569337
Jul 17 03:13:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:53.778Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-ens1f0np0.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689545571179
Jul 17 03:13:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:53.783Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", cpu=\"12\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545569337
Jul 17 03:13:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:53.785Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"tsg_master_mirror_pkts\", app_name=\"tsg_master\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"traffic-engine\", line_name=\"fail\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", table_name=\"flow\", vsysId=\"1\"}" timestamp=1689545567622
Jul 17 03:13:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:53.801Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db21\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545558659
Jul 17 03:13:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:53.804Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_spec_paused\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"local-path-provisioner\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545568950
Jul 17 03:13:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:53.817Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_scrape_collector_success\", asset=\"msh-tsgx12\", asset_id=\"113\", collector=\"bonding\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545563391
Jul 17 03:13:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:53.822Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Running\", pod=\"helm-install-mrzcpd-exporter-4rjwg\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"9b1ef315-33f8-4a3d-a750-cb303d9afe8c\"}" timestamp=1689545568950
Jul 17 03:13:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:53.862Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx12\", asset_id=\"113\", cpu=\"62\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545563391
Jul 17 03:13:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:53.880Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db164\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545570108
Jul 17 03:13:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:53.881Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link_mode\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"vethae353140\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545568328
Jul 17 03:13:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:53.893Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db12\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545570108
Jul 17 03:13:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:53.923Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"auth-rpcgss-module.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"service\"}" timestamp=1689545560884
Jul 17 03:13:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:53.929Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:80-0000:80:01.1-0000:81:00.0-host0-target0:2:0-0:2:0:0-block-sda-sda1.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689545571179
Jul 17 03:13:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:53.941Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-6f587c8cf89e948c5c4c9c6b424be9b08e4c6abf2171e2e5a29c075e4a933263.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"scope\"}" timestamp=1689545560884
Jul 17 03:13:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:53.966Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_inodes_free\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda2\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545550046
Jul 17 03:13:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:53.982Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_sector_writes_total\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda4\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545550046
Jul 17 03:13:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:53.997Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cpu=\"50\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545559409
Jul 17 03:13:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:54.018Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_gc_duration_seconds\", asset=\"pcap-dellr740-dt002\", asset_id=\"33\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"NC-Gohangout-pcap-dellr740-dt002\", endpoint_id=\"173\", module=\"DC-Gohangout\", module_id=\"35\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", quantile=\"0.75\"}" timestamp=1689545572667
Jul 17 03:13:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:54.057Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"msh-tsgx12\", asset_id=\"113\", cpu=\"80\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545563391
Jul 17 03:13:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:54.064Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_waiting\", asset=\"pcap-tsgx07\", asset_id=\"85\", container=\"configmap-reloader\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"configmap-reloader-6c55bbc49f-4sk85\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"1a12ece7-779a-4a70-8faa-cacd80a0de9b\"}" timestamp=1689545562058
Jul 17 03:13:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:54.069Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db39\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545558659
Jul 17 03:13:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:54.076Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_errs_total\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens1f1np1\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545563391
Jul 17 03:13:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:54.124Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db0\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545558659
Jul 17 03:13:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:54.130Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_writes_bytes_total\", asset=\"msh-tsgx12\", asset_id=\"113\", container=\"telegraf\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689545553026
Jul 17 03:13:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:54.171Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_down_changes_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"eno2\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545559409
Jul 17 03:13:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:54.193Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx08\", asset_id=\"65\", cpu=\"60\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545568328
Jul 17 03:13:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:54.208Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_size_bytes\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"shm\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/635ad2e4aebde463bd140e667f3be4804a0be516159f6409fa77a44b96e3f61e/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545568328
Jul 17 03:13:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:54.231Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_gc_duration_seconds\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", quantile=\"0.25\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545558659
Jul 17 03:13:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:54.231Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_ports\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_endpoint=\"cm-policy-local-cache-redis-master\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", port_name=\"tcp-redis\", port_number=\"6379\", port_protocol=\"TCP\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545568663
Jul 17 03:13:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:54.266Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"6\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545563391
Jul 17 03:13:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:54.281Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"tmpfs\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/var/lib/kubelet/pods/5782b922-edbe-45a2-a171-c3d8e1c63480/volumes/kubernetes.io~projected/kube-api-access-gj4zn\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545563391
Jul 17 03:13:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:54.595Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db72\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545558659
Jul 17 03:13:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:54.608Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-a70ba7d6\\\\x2dc83e\\\\x2d453e\\\\x2d962c\\\\x2df10b0073510f-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2dklfkn.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689545571179
Jul 17 03:13:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:54.612Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_labels\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"mrzcpd-exporter\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545568950
Jul 17 03:13:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:55.228Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_resource_requests\", asset=\"pcap-tsgx07\", asset_id=\"85\", container=\"coredns\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", node=\"localhost\", nz_agent_id=\"3\", pod=\"coredns-5f6f6f7d9d-xln7x\", project=\"TSG-OS\", project_id=\"7\", resource=\"memory\", service=\"prometheus-kube-state-metrics\", uid=\"fcb4c20d-46ee-408b-aa8a-c223e2102a3f\", unit=\"byte\"}" timestamp=1689545562058
Jul 17 03:13:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:55.264Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_ips\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", ip=\"10.42.0.82\", ip_family=\"4\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-tsg-traffic-engine-numa-0-clqgr\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"1ea39cdb-8ff7-4292-b608-b56268d85d12\"}" timestamp=1689545562058
Jul 17 03:13:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:55.336Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_created\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"mrzcpd-exporter\", uid=\"c94eaa11-5967-4c23-b255-e589390ba252\"}" timestamp=1689545562058
Jul 17 03:13:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:55.348Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_up_changes_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens3f0np0\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545559409
Jul 17 03:13:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:55.461Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"crond.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689545560884
Jul 17 03:13:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:55.487Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-netns-cni\\\\x2d11799bdf\\\\x2da2e6\\\\x2dcc89\\\\x2d7986\\\\x2dcd4f70ab6d80.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689545560884
Jul 17 03:13:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:56.282Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_usage_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", container=\"dign-server-dns\", cpu=\"total\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545562410
Jul 17 03:13:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:56.308Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"tsg_master_success_log_s\", app_name=\"tsg_master\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"traffic-engine\", line_name=\"VOIP-RECORD\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", table_name=\"sendlog\", vsysId=\"1\"}" timestamp=1689545569194
Jul 17 03:13:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:56.478Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"tmpfs\", name=\"var-lib-kubelet-pods-95e7c99f\\\\x2d2ebd\\\\x2d42b8\\\\x2dabe6\\\\x2d583158385ffe-volumes-kubernetes.io\\\\x7esecret-kubernetes\\\\x2ddashboard\\\\x2dcerts.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689545563372
Jul 17 03:13:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:56.671Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_io_time_weighted_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda2\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545564905
Jul 17 03:13:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:56.687Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_cache\", asset=\"pcap-tsgx03\", asset_id=\"73\", container=\"telegraf-security\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689545566388
Jul 17 03:13:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:56.742Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"96\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545565883
Jul 17 03:13:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:56.765Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_cpu_period\", asset=\"pcap-tsgx03\", asset_id=\"73\", container=\"lb-tcp-80\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545571563
Jul 17 03:13:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:56.856Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"37\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545565883
Jul 17 03:13:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:56.870Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"87\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545565883
Jul 17 03:13:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:57.070Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"pcap-tsgx03\", asset_id=\"73\", condition=\"Available\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"kubernetes-dashboard\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"true\"}" timestamp=1689545573389
Jul 17 03:13:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:57.091Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2dpath-pci\\\\x2d0000:81:00.0\\\\x2dscsi\\\\x2d0:2:0:0\\\\x2dpart5.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689545563372
Jul 17 03:13:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:57.143Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_spec_type\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"kubernetes-dashboard\", type=\"NodePort\", uid=\"882162c5-eb62-4055-bcc0-658a74f30f53\"}" timestamp=1689545573389
Jul 17 03:13:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:57.157Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_labels\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.mrzcpd-exporter.v1\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545573389
Jul 17 03:13:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:57.292Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_flags\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens6f1np1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545565883
Jul 17 03:13:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:57.355Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", container=\"dign-server-dns\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Async\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545562410
Jul 17 03:13:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:57.370Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_colls_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"vethcaaffe87\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545565883
Jul 17 03:13:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:57.428Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"scrape_series_added\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_instance=\"10.42.0.111:9290\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"ipmi-exporter\"}" timestamp=1689545574214
Jul 17 03:13:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:57.447Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689545563372
Jul 17 03:13:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:57.492Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"18\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545565883
Jul 17 03:13:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:57.498Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-pod3c95424c_91e7_4b65_a797_7342b6c9f1ed.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"slice\"}" timestamp=1689545563372
Jul 17 03:13:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:57.522Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_address_assign_type\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth6af7c83b\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545565883
Jul 17 03:13:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:57.538Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled_time\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-node-exporter-gpwdz\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"2ece1d35-4689-4e76-9973-2d85e5294db3\"}" timestamp=1689545573389
Jul 17 03:13:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:57.565Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"auth-rpcgss-module.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"service\"}" timestamp=1689545563372
Jul 17 03:13:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:57.573Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"95\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545565883
Jul 17 03:13:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:57.655Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_complete\", asset=\"pcap-tsgx03\", asset_id=\"73\", condition=\"true\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", job_name=\"helm-install-configmap-reloader\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545573389
Jul 17 03:13:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:57.674Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_owner\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", job_name=\"helm-install-configmap-reloader\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", owner_is_controller=\"<none>\", owner_kind=\"<none>\", owner_name=\"<none>\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545573389
Jul 17 03:13:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:57.690Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_rate_bytes_per_second\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"mlx5_6\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545565883
Jul 17 03:13:58 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:58.977Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_scrape_collector_success\", asset=\"msh-dellr740-dt003\", asset_id=\"9\", collector=\"filesystem\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-dt003\", endpoint_id=\"229\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"hbase_master,clickhouse_data,kafka,gohangout,zookeeper,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545577407
Jul 17 03:13:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:59.222Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_memory_AnonHugePages_bytes\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"OLAP-Node-Exporter-pcap-dellr740-dt001\", endpoint_id=\"251\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545577558
Jul 17 03:13:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:59.288Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", cpu=\"15\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"OLAP-Node-Exporter-pcap-dellr740-dt001\", endpoint_id=\"251\", mode=\"steal\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545577558
Jul 17 03:13:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:59.700Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"119\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545569083
Jul 17 03:13:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:59.720Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"108\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545569083
Jul 17 03:13:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:59.861Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-45b995ae\\\\x2d405c\\\\x2d4e5d\\\\x2db21a\\\\x2d3830d4e78e4d-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2d27shn.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689545564879
Jul 17 03:13:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:13:59.906Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-b3efc35a\\\\x2d4844\\\\x2d47d9\\\\x2d82c9\\\\x2d7d335d0d4eaa-volume\\\\x2dsubpaths-config\\\\x2dvolume-firewall-1.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689545564879
Jul 17 03:14:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:00.055Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_file_descriptors\", asset=\"msh-tsgx01\", asset_id=\"51\", container=\"telegraf-security\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689545559115
Jul 17 03:14:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:00.059Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx01\", asset_id=\"51\", container=\"kubernetes-dashboard\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Sync\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545562764
Jul 17 03:14:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:00.308Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_ready\", asset=\"msh-tsgx01\", asset_id=\"51\", container=\"metrics\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"cm-policy-local-cache-redis-master-69967b5879-kmxnw\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"afda43ab-d2e7-4daa-bc44-9edf3c57be96\"}" timestamp=1689545567425
Jul 17 03:14:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:00.328Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_observed_generation\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"local-path-provisioner\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545567425
Jul 17 03:14:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:00.341Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"106\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545569083
Jul 17 03:14:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:00.343Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_restart_policy\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-kube-state-metrics-6b768c7bf8-fnk7m\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", type=\"Always\", uid=\"75a1cd69-272c-4ee3-afeb-f27cbbc02353\"}" timestamp=1689545567425
Jul 17 03:14:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:00.398Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545569083
Jul 17 03:14:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:00.413Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"77\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545569083
Jul 17 03:14:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:00.711Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db233\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545565071
Jul 17 03:14:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:00.712Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_ulimits_soft\", asset=\"msh-tsgx01\", asset_id=\"51\", container=\"local-path-provisioner\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", ulimit=\"max_open_files\"}" timestamp=1689545565616
Jul 17 03:14:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:00.725Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_netstat_Tcp_OutSegs\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545569083
Jul 17 03:14:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:00.726Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sssd-kcm.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689545564879
Jul 17 03:14:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:00.748Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"33\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545569083
Jul 17 03:14:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:00.791Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db248\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545565071
Jul 17 03:14:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:00.840Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_packets_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"eno36np3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545569083
Jul 17 03:14:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:00.857Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dracut-initqueue.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"service\"}" timestamp=1689545564879
Jul 17 03:14:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:01.028Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"msh-tsgx01\", asset_id=\"51\", condition=\"Progressing\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"cm-policy-local-cache-redis-master\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"unknown\"}" timestamp=1689545567425
Jul 17 03:14:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:02.373Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"msh-tsgx02\", asset_id=\"63\", cpu=\"54\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545578318
Jul 17 03:14:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:02.395Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"overlay\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-7826a883f20bbd62b0d475cfe7dd1a69d6eabca2f974e5a1583a0b712d5bfa3b-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689545568447
Jul 17 03:14:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:02.518Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-mrzcpd-exporter-lvwd9\", project=\"TSG-OS\", project_id=\"7\", reason=\"Shutdown\", service=\"prometheus-kube-state-metrics\", uid=\"225625d7-83ea-4ba9-8029-a534c5c1fb0c\"}" timestamp=1689545571412
Jul 17 03:14:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:02.555Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_errors_received_total\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_1\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545578318
Jul 17 03:14:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:02.614Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_packets_total\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"eno2\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545578318
Jul 17 03:14:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:02.621Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"nfsdcld.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"service\"}" timestamp=1689545568447
Jul 17 03:14:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:02.656Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"msh-tsgx02\", asset_id=\"63\", container=\"telegraf\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", state=\"iowaiting\"}" timestamp=1689545561840
Jul 17 03:14:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:02.736Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_info\", asset=\"msh-tsgx02\", asset_id=\"63\", configmap=\"kube-root-ca.crt\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-public\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545571412
Jul 17 03:14:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:02.750Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_labels\", asset=\"msh-tsgx02\", asset_id=\"63\", configmap=\"chart-content-tsg-traffic-engine-numa-1\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545571412
Jul 17 03:14:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:02.871Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", phase=\"Unknown\", pod=\"dashboard-metrics-scraper-5b94897dc9-gkbdq\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"03938281-e348-4392-a795-f2d53f05d546\"}" timestamp=1689545571412
Jul 17 03:14:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:02.883Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"msh-tsgx02\", asset_id=\"63\", condition=\"false\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-delete-tsg-traffic-engine-numa-1-ddm55\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"de0dc5f3-860f-415f-8f70-95817cf0299d\"}" timestamp=1689545571412
Jul 17 03:14:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:02.973Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_rdb_last_bgsave_duration_sec\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545569269
Jul 17 03:14:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:03.104Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_annotations\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_endpoint=\"cm-policy-local-cache-redis-headless\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545571412
Jul 17 03:14:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:03.104Z caller=dedupe.go:112 component=remote level=info remote_name=52db6f url=http://10.10.10.159:10090/nz-agent/prometheus/proxy/api/v1/write msg="Dropped sample for series that was not explicitly dropped via relabelling" ref=4436650
Jul 17 03:14:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:03.118Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_restarts_total\", asset=\"msh-tsgx02\", asset_id=\"63\", container=\"helm\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-prometheus-glw8n\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"bc80432c-841c-4177-817a-bf841f359b9d\"}" timestamp=1689545571412
Jul 17 03:14:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:03.133Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"msh-tsgx02\", asset_id=\"63\", cpu=\"121\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545578318
Jul 17 03:14:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:03.136Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx02\", asset_id=\"63\", cpu=\"53\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545578318
Jul 17 03:14:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:03.160Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db74\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545569269
Jul 17 03:14:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:03.213Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-infiniband-umad3.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689545568447
Jul 17 03:14:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:03.230Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"net_conntrack_dialer_conn_failed_total\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", dialer_name=\"221\", instance=\"10.10.20.159:10090\", reason=\"timeout\", type=\"global\"}" timestamp=1689545582990
Jul 17 03:14:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:03.339Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-virtual-net-veth14e44624.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689545568447
Jul 17 03:14:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:03.419Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_running\", asset=\"msh-tsgx02\", asset_id=\"63\", container=\"dashboard-metrics-scraper\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", pod=\"dashboard-metrics-scraper-5b94897dc9-gkbdq\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"03938281-e348-4392-a795-f2d53f05d546\"}" timestamp=1689545571412
Jul 17 03:14:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:03.456Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_reservation_limit_bytes\", asset=\"msh-tsgx02\", asset_id=\"63\", container=\"dign-server-web\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545572003
Jul 17 03:14:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:03.489Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_metadata_generation\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"prometheus-pushgateway-bd476bcf9\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545571412
Jul 17 03:14:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:03.625Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"23\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545578318
Jul 17 03:14:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:03.643Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_cpu_shares\", asset=\"msh-tsgx02\", asset_id=\"63\", container=\"dign-client\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545572003
Jul 17 03:14:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:03.643Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx02\", asset_id=\"63\", cpu=\"1\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545578318
Jul 17 03:14:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:03.663Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_info\", asset=\"msh-tsgx02\", asset_id=\"63\", container=\"helm\", container_id=\"containerd://cd3b6ce92eba3959eecae3e5e65890f7307638e1b00b521da24b956f8e6a0527\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", image=\"docker.io/rancher/klipper-helm:v0.8.0-build20230510\", image_id=\"sha256:6f42df210d7faa0b6eb56f5f28bb061930c7834af1e7dd4b3586108ec6b1485b\", image_spec=\"rancher/klipper-helm:v0.8.0-build20230510\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-traefik-crd-99hbv\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"10c928ff-b1be-480e-97e1-363fb468914b\"}" timestamp=1689545571412
Jul 17 03:14:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:03.701Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx02\", asset_id=\"63\", cpu=\"79\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545578318
Jul 17 03:14:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:03.716Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx02\", asset_id=\"63\", cpu=\"0\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545578318
Jul 17 03:14:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:03.756Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_node_status_allocatable\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", node=\"localhost\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", resource=\"hugepages_2Mi\", service=\"prometheus-kube-state-metrics\", unit=\"byte\"}" timestamp=1689545571412
Jul 17 03:14:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:03.774Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_ips\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", ip=\"10.42.0.138\", ip_family=\"4\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"local-path-provisioner-957fdf8bc-dnfth\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"a393199f-26cc-4dc0-ab08-cd30ed8356d2\"}" timestamp=1689545571412
Jul 17 03:14:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:03.785Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_device_id\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"docker0\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-dt001\", endpoint_id=\"225\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545582988
Jul 17 03:14:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:03.839Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"prometheus_sd_discovered_targets\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", config=\"135\", datacenter=\"MSH\", instance=\"10.10.20.159:10090\", name=\"scrape\", type=\"global\"}" timestamp=1689545582990
Jul 17 03:14:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:03.843Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"net_conntrack_dialer_conn_failed_total\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", dialer_name=\"asset_ping_119\", instance=\"10.10.20.159:10090\", reason=\"refused\", type=\"global\"}" timestamp=1689545582990
Jul 17 03:14:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:04.049Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"net_conntrack_dialer_conn_failed_total\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", dialer_name=\"213\", instance=\"10.10.20.159:10090\", reason=\"unknown\", type=\"global\"}" timestamp=1689545582990
Jul 17 03:14:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:04.465Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"prometheus_target_scrapes_exceeded_sample_limit_total\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", instance=\"10.10.20.159:10090\", type=\"global\"}" timestamp=1689545582990
Jul 17 03:14:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:06.602Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_id\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth747cb07e\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545583910
Jul 17 03:14:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:06.610Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-e96787f3\\\\x2d8c20\\\\x2d4c5e\\\\x2da395\\\\x2dec874ca09072-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2dlg9hn.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689545573582
Jul 17 03:14:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:07.019Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-34ba848e05d4ba2a53edc9114a2a216d54aa396606f7b4fb7d113518104d40b5-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689545573582
Jul 17 03:14:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:07.043Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-sysusers.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689545573582
Jul 17 03:14:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:07.461Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_swap\", asset=\"twa-tsgx01\", asset_id=\"91\", container=\"dashboard-metrics-scraper\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545575739
Jul 17 03:14:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:07.966Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"126\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545583910
Jul 17 03:14:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:07.987Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cpu=\"3\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545583910
Jul 17 03:14:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:08.006Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"clickhouse_interserver_connection\", asset=\"msh-dellr740-olap002\", asset_id=\"13\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Clickhouse-msh-dellr740-olap002\", endpoint_id=\"73\", module=\"NC-Clickhouse\", module_id=\"29\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545587759
Jul 17 03:14:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:08.074Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_oom_events_total\", asset=\"twa-tsgx01\", asset_id=\"91\", container=\"dign-server-web\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545568785
Jul 17 03:14:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:08.097Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-9b7a7ee8\\\\x2d3d4c\\\\x2d4d13\\\\x2db2bb\\\\x2dd0637b89d0c8-volume\\\\x2dsubpaths-telegraf-telegraf-0.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689545573582
Jul 17 03:14:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:08.178Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cpu=\"32\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545583910
Jul 17 03:14:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:08.207Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx01\", asset_id=\"91\", container=\"firewall\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Sync\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689545570857
Jul 17 03:14:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:08.591Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"clickhouse_os_soft_irq_time_cpu34\", asset=\"msh-dellr740-olap002\", asset_id=\"13\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Clickhouse-msh-dellr740-olap002\", endpoint_id=\"73\", module=\"NC-Clickhouse\", module_id=\"29\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545587759
Jul 17 03:14:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:08.745Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"thanos_objstore_bucket_operation_duration_seconds_bucket\", agent_name=\"msh-kvm-nz001\", application=\"cortex\", component=\"querier\", datacenter=\"MSH\", instance=\"127.0.0.1:10093\", le=\"0.3\", operation=\"attributes\"}" timestamp=1689545587261
Jul 17 03:14:09 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:09.122Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"clickhouse_block_queue_time_sdb\", asset=\"msh-dellr740-olap002\", asset_id=\"13\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Clickhouse-msh-dellr740-olap002\", endpoint_id=\"73\", module=\"NC-Clickhouse\", module_id=\"29\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545587759
Jul 17 03:14:09 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:09.606Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_replicaid\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"assessment_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"38e639c1ffb12e33444c5eeed46e9713\", table=\"index_filename_assessment_hos_bucket\", type=\"index\"}" timestamp=1689545588804
Jul 17 03:14:09 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:09.615Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_deleteCount\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"a3eb769a8c2d325e0a97838f6786a569\", table=\"session_record_hos_bucket\", type=\"data\"}" timestamp=1689545588804
Jul 17 03:14:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:10.321Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_deleteCount\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"proxy_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"2ea4c2e074dbea4bc82df3819be3b6ce\", table=\"index_time_proxy_hos_bucket\", type=\"index\"}" timestamp=1689545588804
Jul 17 03:14:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:10.607Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_FlushTime_mean\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Server\"}" timestamp=1689545588804
Jul 17 03:14:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:11.405Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_4_metric_getTime_mean\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545590148
Jul 17 03:14:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:11.423Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_194_2_metric_storeCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545590148
Jul 17 03:14:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:12.026Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_1_metric_incrementTime_99_9th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545590148
Jul 17 03:14:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:12.027Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_MajorCompactionOutputFileCount_median\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Server\"}" timestamp=1689545590148
Jul 17 03:14:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:12.186Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_putCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"7bb76a6de6c05cfd2977efe16321e390\", table=\"default\", type=\"data\"}" timestamp=1689545590148
Jul 17 03:14:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:12.197Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"java_lang_OperatingSystem_TotalPhysicalMemorySize\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545590148
Jul 17 03:14:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:12.199Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_maxStoreFileAge\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"knowledge_base_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"13a4e6025bf464f6e219ed4362f86f50\", table=\"index_time_knowledge_base_hos_bucket\", type=\"index\"}" timestamp=1689545590148
Jul 17 03:14:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:12.711Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_scanCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"96a55ea6420b424b55926f18a013afc6\", table=\"firewall_hos_bucket\", type=\"data\"}" timestamp=1689545590148
Jul 17 03:14:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:12.863Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_deleteCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"rtp_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"b25bc8b7b58f2fe2c8ddaad6e44f85fc\", table=\"index_filename_rtp_hos_bucket\", type=\"index\"}" timestamp=1689545590148
Jul 17 03:14:13 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:13.090Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_session_record_hos_bucket_metric_deleteTime_mean\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545590148
Jul 17 03:14:13 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:13.136Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_compactionsFailedCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"session_record_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"ded3a29a944bd173f3f8756c8f131314\", table=\"index_time_session_record_hos_bucket\", type=\"index\"}" timestamp=1689545590148
Jul 17 03:14:13 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:13.314Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_FlushTime_99_9th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Server\"}" timestamp=1689545590148
Jul 17 03:14:13 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:13.395Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_avgStoreFileAge\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"firewall_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"963d87c0ad507a8707a36cf6f228f215\", table=\"index_partfile_firewall_hos_bucket\", type=\"index\"}" timestamp=1689545590148
Jul 17 03:14:13 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:13.445Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_SplitTime_min\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Server\"}" timestamp=1689545590148
Jul 17 03:14:13 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:13.505Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_MUTEX_metric_splitTime_75th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545590148
Jul 17 03:14:13 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:13.764Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_session_record_hos_bucket_metric_compactionTime_95th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545590148
Jul 17 03:14:13 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:13.983Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_memStoreSize\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"rtp_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"afda67220314ac681ee8f19c890bcc36\", table=\"index_partfile_rtp_hos_bucket\", type=\"index\"}" timestamp=1689545590148
Jul 17 03:14:14 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:14.024Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_deleteCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"0bb1bcaafc52c55bd933ad4c1c82d678\", table=\"session_record_hos_bucket\", type=\"data\"}" timestamp=1689545590148
Jul 17 03:14:14 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:14.061Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_LOG_metric_deleteBatchTime_99th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545590148
Jul 17 03:14:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:16.192Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_reservation_limit_bytes\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689545589150
Jul 17 03:14:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:16.210Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"dashboard-metrics-scraper\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545588076
Jul 17 03:14:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:16.248Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_free_bytes\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"shm\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/2460251b55f7a06a933f02003bcdf1cd1beb307337aa042624eac5a55a2549a6/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545581230
Jul 17 03:14:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:16.263Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-f23fca0a43025076302063f94e9d6892b79f439f990ab3a43d60d379668367e3-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689545587046
Jul 17 03:14:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:16.337Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:e0-0000:e0:03.1-0000:e2:00.1-net-ens8f1np1.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689545587046
Jul 17 03:14:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:16.356Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-163bbbd243d482fb7f2d7d2ce77a7dad5c82c7ba6f818f38ef581f255a1df525-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689545587046
Jul 17 03:14:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:16.356Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx07\", asset_id=\"103\", container=\"dign-server-dns\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Sync\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545576436
Jul 17 03:14:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:16.366Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"14\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545581230
Jul 17 03:14:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:16.459Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:80-0000:80:01.2-0000:82:00.0-net-ens4f0np0.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689545587046
Jul 17 03:14:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:16.562Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_dormant\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"eno35np2\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545581230
Jul 17 03:14:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:16.574Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_net_dev_group\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"br_dign_c\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545581230
Jul 17 03:14:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:16.615Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"process_open_fds\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545586027
Jul 17 03:14:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:16.636Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_forks_total\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545581230
Jul 17 03:14:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:16.656Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx07\", asset_id=\"103\", container=\"dign-client\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Async\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545576628
Jul 17 03:14:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:16.754Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_discards_transmitted_total\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_4\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545581230
Jul 17 03:14:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:16.770Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethe34782c1\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545581230
Jul 17 03:14:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:16.841Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-rfkill.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"service\"}" timestamp=1689545587046
Jul 17 03:14:18 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:18.518Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"clickhouse_network_receive_bytes_em4\", asset=\"twa-dellr740-dt003\", asset_id=\"41\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"NC-Clickhouse-twa-dellr740-dt003\", endpoint_id=\"91\", module=\"NC-Clickhouse\", module_id=\"29\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545598300
Jul 17 03:14:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:19.420Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-460f5be153f52052cd29535bc0132c4b771599ff123f1c6a334c8908afcd72a4.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"scope\"}" timestamp=1689545584356
Jul 17 03:14:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:19.905Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_usage_bytes\", asset=\"twa-tsgx06\", asset_id=\"101\", container=\"dign-server-dns\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545585380
Jul 17 03:14:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:19.921Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_persistentvolume_created\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", persistentvolume=\"pvc-c5e91f11-de8e-4cf4-bc30-03d675e8fed4\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545593949
Jul 17 03:14:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:20.292Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_daemonset_metadata_generation\", asset=\"twa-tsgx06\", asset_id=\"101\", daemonset=\"dign-client\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545593949
Jul 17 03:14:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:20.343Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_writes_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda4\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545588517
Jul 17 03:14:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:20.365Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_mapped_file\", asset=\"pcap-tsgx10\", asset_id=\"89\", container=\"local-path-provisioner\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545589198
Jul 17 03:14:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:20.453Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_changes_total\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens6f1np1\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545597612
Jul 17 03:14:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:20.611Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"mlocate-updatedb.timer\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"timer\"}" timestamp=1689545584356
Jul 17 03:14:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:20.903Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_mapped_file\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689545588229
Jul 17 03:14:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:21.085Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"56\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545588263
Jul 17 03:14:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:21.115Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Udp_Bit\", app_name=\"sapp4\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", vsysId=\"1\"}" timestamp=1689545599077
Jul 17 03:14:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:21.250Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_edac_uncorrectable_errors_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", controller=\"1\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545588263
Jul 17 03:14:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:21.273Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_avail_bytes\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"shm\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/9fbd3ff04dd48db4b39f55e3c9158c39d3277e82aad9856f80daed4ca7246221/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545588263
Jul 17 03:14:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:21.635Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_running\", asset=\"pcap-tsgx10\", asset_id=\"89\", container=\"prometheus-node-exporter\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-node-exporter-kd42j\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"3babe650-f538-4383-8bda-a77b2daf72dc\"}" timestamp=1689545593828
Jul 17 03:14:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:21.740Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_readonly\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"shm\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/93c319d73ae73405c0e05638ab02770e97a7c13d0da4dea58199e6edcf24a334/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545588263
Jul 17 03:14:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:22.276Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", cpu=\"21\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"OLAP-Node-Exporter-twa-dellr740-dt001\", endpoint_id=\"257\", mode=\"user\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545601405
Jul 17 03:14:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:22.298Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_next_gc_bytes\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"OLAP-Node-Exporter-twa-dellr740-dt001\", endpoint_id=\"257\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545601405
Jul 17 03:14:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:22.321Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_frequency_max_hertz\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", cpu=\"27\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"OLAP-Node-Exporter-twa-dellr740-dt001\", endpoint_id=\"257\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545601405
Jul 17 03:14:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:22.789Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_gc_duration_seconds_count\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"OLAP-Node-Exporter-twa-dellr740-dt001\", endpoint_id=\"257\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545601405
Jul 17 03:14:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:22.804Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_scaling_frequency_min_hertz\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", cpu=\"5\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"OLAP-Node-Exporter-twa-dellr740-dt001\", endpoint_id=\"257\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545601405
Jul 17 03:14:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:23.075Z caller=dedupe.go:112 component=remote level=error remote_name=52db6f url=http://10.10.10.159:10090/nz-agent/prometheus/proxy/api/v1/write msg="non-recoverable error" count=1000 exemplarCount=0 err="server returned HTTP status 400 : out of order sample"
Jul 17 03:14:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:23.552Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_spec_replicas\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"local-path-provisioner\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545600068
Jul 17 03:14:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:23.570Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_start_time\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-ipmi-exporter-v8l8s\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"a3ae0996-f357-4355-bdfb-2d634555c081\"}" timestamp=1689545600068
Jul 17 03:14:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:23.617Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx09\", asset_id=\"55\", cpu=\"114\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545594790
Jul 17 03:14:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:23.636Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-vethee323c11.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689545598416
Jul 17 03:14:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:23.651Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_tasks_max\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dracut-initqueue.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689545598416
Jul 17 03:14:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:23.674Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-405e3b50\\\\x2d5fa3\\\\x2d4f39\\\\x2d9405\\\\x2dac337f946135-volume\\\\x2dsubpaths-telegraf-telegraf\\\\x2dsecurity-0.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689545598416
Jul 17 03:14:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:23.693Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2dpath-pci\\\\x2d0000:81:00.0\\\\x2dscsi\\\\x2d0:2:0:0.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689545598416
Jul 17 03:14:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:23.708Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_packets_dropped_total\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", interface=\"eth0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689545588600
Jul 17 03:14:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:23.718Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"iscsi-onboot.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689545598416
Jul 17 03:14:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:23.814Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-pod4c92ea3a_4155_49d3_9d36_59027e09d717.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"slice\"}" timestamp=1689545598416
Jul 17 03:14:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:23.837Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.grpc.v1.cri-sandboxes-badc6d80be8fc6a17c57d479d7af9ac71f403d1b8ddf9fa49ecca8350872cc4f-shm.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689545598416
Jul 17 03:14:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:23.843Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_local_link_integrity_errors_total\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_7\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545594790
Jul 17 03:14:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:24.153Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"17\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545600160
Jul 17 03:14:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:24.157Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"remote-fs-pre.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"target\"}" timestamp=1689545598416
Jul 17 03:14:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:24.179Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_flags\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens6f0np0\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545594790
Jul 17 03:14:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:24.196Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_allocator_frag_ratio\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545597019
Jul 17 03:14:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:24.261Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"78\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545600160
Jul 17 03:14:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:24.278Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"3\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545600160
Jul 17 03:14:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:24.332Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"pcap-tsgx12\", asset_id=\"117\", condition=\"unknown\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"kube-multus-ds-dsd65\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"e9990f8b-7450-47a8-867f-7387f0ac7e25\"}" timestamp=1689545591122
Jul 17 03:14:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:24.377Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", phase=\"Succeeded\", pod=\"ipmi-exporter-6f8464dd94-n7w69\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"9e233e9f-de40-4ebd-abfb-e6360a2d9bb6\"}" timestamp=1689545600068
Jul 17 03:14:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:24.399Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-4929687ad054d95ab25d92b84929eaadafda461c68f8ccbb095b8c61bcabb765.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"scope\"}" timestamp=1689545598416
Jul 17 03:14:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:24.418Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2did-scsi\\\\x2d362cea7f0848263002c2c205714e9cdc4\\\\x2dpart5.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689545598416
Jul 17 03:14:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:24.422Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"94\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545600160
Jul 17 03:14:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:24.590Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"kube-state-metrics\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Discard\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545597284
Jul 17 03:14:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:24.595Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"msh-tsgx09\", asset_id=\"55\", cpu=\"13\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545594790
Jul 17 03:14:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:24.623Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_speed_bytes\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth6334ec18\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545600160
Jul 17 03:14:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:24.637Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-pod4c92ea3a_4155_49d3_9d36_59027e09d717.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"slice\"}" timestamp=1689545598416
Jul 17 03:14:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:24.640Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"coredns\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Async\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545596093
Jul 17 03:14:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:24.715Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db83\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545597019
Jul 17 03:14:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:24.748Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"abrt-ccpp.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"service\"}" timestamp=1689545598416
Jul 17 03:14:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:24.750Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_errors_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", interface=\"eno35np2\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545602030
Jul 17 03:14:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:24.814Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"pcap-tsgx12\", asset_id=\"117\", condition=\"unknown\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"coredns-ff8b8fd9d-c4wjl\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"a6dafded-0d01-479f-906c-089305c5928a\"}" timestamp=1689545591122
Jul 17 03:14:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:25.021Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-infiniband-umad9.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689545598416
Jul 17 03:14:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:25.076Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-0cf608ff3c2bd8a7cc8787a4affd9116fa4d607598344d94c1c75a9e02a8481f.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"scope\"}" timestamp=1689545598416
Jul 17 03:14:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:25.153Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-405e3b50\\\\x2d5fa3\\\\x2d4f39\\\\x2d9405\\\\x2dac337f946135-volume\\\\x2dsubpaths-telegraf-telegraf-0.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689545598416
Jul 17 03:14:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:25.376Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"atd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"service\"}" timestamp=1689545591102
Jul 17 03:14:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:25.659Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"java_lang_OperatingSystem_SystemLoadAverage\", asset=\"pcap-dellr740-dt003\", asset_id=\"35\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-Yarn-pcap-dellr740-dt003-NodeManager\", endpoint_id=\"361\", module=\"DC-Yarn\", module_id=\"1\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545605525
Jul 17 03:14:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:26.017Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_daemonset_status_number_ready\", asset=\"pcap-tsgx12\", asset_id=\"117\", daemonset=\"svclb-traefik-05ed3a27\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545591122
Jul 17 03:14:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:26.340Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-traefik-vw4k2\", project=\"TSG-OS\", project_id=\"7\", reason=\"UnexpectedAdmissionError\", service=\"prometheus-kube-state-metrics\", uid=\"f01b2d19-81c4-4465-a1a9-08cbc2270574\"}" timestamp=1689545598744
Jul 17 03:14:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:26.380Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_link_error_recovery_total\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_9\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545595810
Jul 17 03:14:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:26.507Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_NodeManager_HeartbeatAvgTime\", asset=\"pcap-dellr740-dt003\", asset_id=\"35\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-Yarn-pcap-dellr740-dt003-NodeManager\", endpoint_id=\"361\", module=\"DC-Yarn\", module_id=\"1\", name=\"RpcDetailedActivityForPort8040\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545605525
Jul 17 03:14:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:26.537Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"44\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545595810
Jul 17 03:14:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:26.579Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_flags\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"virtio_dign_c\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545595810
Jul 17 03:14:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:26.596Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-ask-password-wall.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689545593907
Jul 17 03:14:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:26.769Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"77\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545595810
Jul 17 03:14:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:26.811Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_running\", asset=\"twa-tsgx09\", asset_id=\"107\", container=\"prometheus-pushgateway\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-pushgateway-bd476bcf9-zqlv8\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"880aa12b-51b0-4413-88cb-d06b2fc6d0c3\"}" timestamp=1689545598744
Jul 17 03:14:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:26.823Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_fifo_total\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"br_dign_s\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545595810
Jul 17 03:14:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:26.925Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_mspan_inuse_bytes\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"ipmi-exporter\"}" timestamp=1689545601597
Jul 17 03:14:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:26.937Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"73\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545595810
Jul 17 03:14:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:26.950Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_cpu_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-pod1d1df77f_66ac_4156_8ea3_66e8a287970d.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"slice\"}" timestamp=1689545593907
Jul 17 03:14:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:26.982Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Running\", pod=\"helm-install-traefik-crd-h8g7g\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"e303f019-5869-4dc2-afe6-c1107cceb13d\"}" timestamp=1689545598744
Jul 17 03:14:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:27.026Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_stack_inuse_bytes\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545603549
Jul 17 03:14:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:27.076Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"21\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545595810
Jul 17 03:14:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:27.507Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_limit_bytes\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"lb-tcp-80\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545604841
Jul 17 03:14:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:27.510Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"jvm_memory_pool_bytes_used\", asset=\"pcap-dellr740-dt002\", asset_id=\"33\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt002-Hmaster\", endpoint_id=\"345\", module=\"DC-HBase\", module_id=\"3\", nz_agent_id=\"3\", pool=\"CMS Old Gen\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545607137
Jul 17 03:14:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:27.565Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_ports\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_endpoint=\"kube-dns\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", port_name=\"metrics\", port_number=\"9153\", port_protocol=\"TCP\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545595682
Jul 17 03:14:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:27.567Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"33\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545602882
Jul 17 03:14:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:27.584Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_info\", address=\"08:c0:eb:53:c4:79\", asset=\"msh-tsgx10\", asset_id=\"69\", broadcast=\"ff:ff:ff:ff:ff:ff\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens1f1np1\", duplex=\"full\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operstate=\"up\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545602882
Jul 17 03:14:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:27.595Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_waiting\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"helm\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-tsg-traffic-engine-numa-0-dt6k8\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"3456946b-cbfb-47c0-ab79-75794f863687\"}" timestamp=1689545595682
Jul 17 03:14:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:27.608Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"6\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545602882
Jul 17 03:14:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:27.629Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_labels\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"systemd-exporter-7f549d9485-9zshn\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"163ef19a-c78d-47c5-82c4-3bf5914611f1\"}" timestamp=1689545595682
Jul 17 03:14:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:27.645Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:a0-0000:a0:03.1-0000:a1:00.0-net-ens6f0np0.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689545594114
Jul 17 03:14:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:27.694Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"98\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545602882
Jul 17 03:14:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:27.715Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_labels\", asset=\"msh-tsgx10\", asset_id=\"69\", configmap=\"extension-apiserver-authentication\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545595682
Jul 17 03:14:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:27.751Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"16\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545602882
Jul 17 03:14:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:28.107Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_restarts_total\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"firewall\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", pod=\"tsg-traffic-engine-numa-0-firewall-556f86477-7wv6g\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"0b0292d1-7c6a-4fad-a1f9-f14e3c7cb4f8\"}" timestamp=1689545595682
Jul 17 03:14:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:28.284Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"36\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545602882
Jul 17 03:14:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:28.310Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_commands_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cmd=\"latency\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545599446
Jul 17 03:14:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:28.391Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db56\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545599446
Jul 17 03:14:29 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:29.198Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"20\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545602882
Jul 17 03:14:29 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:29.212Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"28\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545602882
Jul 17 03:14:29 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:29.359Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_device_id\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth875ea1eb\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545602882
Jul 17 03:14:29 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:29.972Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"pcap-tsgx02\", asset_id=\"75\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db43\", endpoint=\"TSG-OS-pcap-tsgx02\", endpoint_id=\"469\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545605681
Jul 17 03:14:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:31.204Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas_available\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"tsg-traffic-engine-numa-0-firewall\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545604068
Jul 17 03:14:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:31.521Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_daemonset_created\", asset=\"msh-tsgx05\", asset_id=\"67\", daemonset=\"dign-server\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545604068
Jul 17 03:14:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:31.580Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db179\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545610141
Jul 17 03:14:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:31.649Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_system_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", container=\"firewall\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689545601910
Jul 17 03:14:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:31.668Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_node_status_condition\", asset=\"twa-tsgx03\", asset_id=\"95\", condition=\"Ready\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", node=\"localhost\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"unknown\"}" timestamp=1689545610436
Jul 17 03:14:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:31.668Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_metadata_resource_version\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.configmap-reloader.v1\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545604068
Jul 17 03:14:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:31.672Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_spec_completions\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", job_name=\"helm-install-ipmi-exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545610436
Jul 17 03:14:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:31.693Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:00-0000:00:01.1-0000:01:00.1-net-ens1f1np1.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689545606907
Jul 17 03:14:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:31.746Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"42\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545600791
Jul 17 03:14:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:31.769Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"irqbalance.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"service\"}" timestamp=1689545602644
Jul 17 03:14:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:31.782Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"25\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545599093
Jul 17 03:14:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:31.962Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_spec_paused\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"prometheus-pushgateway\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545604068
Jul 17 03:14:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:31.986Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", phase=\"Running\", pod=\"cm-policy-local-cache-redis-master-69967b5879-rj8lj\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"adf25328-be5b-4a8f-99a7-ca39872df08e\"}" timestamp=1689545604068
Jul 17 03:14:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:32.137Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-pod0f76af51_4cb4_40c5_8ff5_c445f8b7b608.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"slice\"}" timestamp=1689545606907
Jul 17 03:14:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:32.153Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-2096ff4a\\\\x2d92d2\\\\x2d47a7\\\\x2d9b70\\\\x2d293a6568cee4-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2d8tn7x.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689545602644
Jul 17 03:14:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:32.297Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"auditd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"forking\", type=\"service\"}" timestamp=1689545602644
Jul 17 03:14:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:32.318Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_info\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", job_name=\"helm-install-tsg-traffic-engine-numa-1\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545599276
Jul 17 03:14:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:32.339Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"59\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545600791
Jul 17 03:14:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:32.364Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"109\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545600939
Jul 17 03:14:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:32.450Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_created\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_endpoint=\"ipmi-exporter\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545599894
Jul 17 03:14:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:32.470Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"85\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545600605
Jul 17 03:14:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:32.527Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_bytes_total\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth30181fe0\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545600939
Jul 17 03:14:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:32.557Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"nsfs\", name=\"run-netns-cni\\\\x2d58890221\\\\x2d3738\\\\x2dd048\\\\x2d501c\\\\x2d9bf6742bc7d2.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689545601015
Jul 17 03:14:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:32.574Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_completion_time\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-ipmi-exporter-qg4pr\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"6d488923-85e6-4a2b-a06b-4a00d69292b7\"}" timestamp=1689545599894
Jul 17 03:14:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:32.593Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_files\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"shm\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/88f0e989522bb3cf4f957acb649de39004e023b0b1747809d0eafd8f5eb5f572/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545599093
Jul 17 03:14:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:32.610Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_netstat_Icmp6_OutMsgs\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545600939
Jul 17 03:14:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:32.654Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_drop_total\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethc3a959fd\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545600605
Jul 17 03:14:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:32.673Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"102\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545600605
Jul 17 03:14:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:32.697Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"33\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545600791
Jul 17 03:14:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:32.702Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_bytes_total\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens3f0np0\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545600939
Jul 17 03:14:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:32.713Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-ffc561afd76df565091113b67bd7f7bd0747f00a360ec023d1dc9236d7f6d2a7.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"scope\"}" timestamp=1689545602644
Jul 17 03:14:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:32.721Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-823a9850cf952fec07f6e1b9e107f7e86acacfac1e6dcbb4539307bed767287f.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"scope\"}" timestamp=1689545606907
Jul 17 03:14:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:32.852Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db184\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545596711
Jul 17 03:14:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:32.876Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_restarts_total\", asset=\"twa-tsgx05\", asset_id=\"99\", container=\"prometheus-server-configmap-reload\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-server-6ff784d758-4hdpm\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"bfa69e6a-60ec-4e46-b7d7-27a864a8d19e\"}" timestamp=1689545599894
Jul 17 03:14:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:32.940Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545600605
Jul 17 03:14:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:32.956Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_waiting\", asset=\"twa-tsgx05\", asset_id=\"99\", container=\"kube-state-metrics\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-kube-state-metrics-6b768c7bf8-h2rv4\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"37526140-01d6-46f5-9f0b-e066a6254bac\"}" timestamp=1689545599894
Jul 17 03:14:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:33.005Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-9fec12b63663a4b4c3edc8da0ea94d0f53bed6ca984ae78b45fed0d7e9985c3a-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689545607691
Jul 17 03:14:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:33.016Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_processes\", asset=\"twa-tsgx03\", asset_id=\"95\", container=\"ipmi-exporter\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545597907
Jul 17 03:14:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:33.092Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-4425f2c0a7cf12a1a542e67431b564f1340cea0beb326affce132a24e3c6e4e4-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689545602644
Jul 17 03:14:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:33.107Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"twa-tsgx03\", asset_id=\"95\", condition=\"false\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-configmap-reloader-p2p4j\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"f81c8fb3-e0bf-42af-a2a5-7e8f904d8466\"}" timestamp=1689545610436
Jul 17 03:14:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:33.207Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_fully_labeled_replicas\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-0-firewall-768dc68f5b\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545599894
Jul 17 03:14:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:33.228Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"scrape_series_added\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_instance=\"10.10.30.165:9008\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689545607691
Jul 17 03:14:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:33.245Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"7\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545599093
Jul 17 03:14:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:33.414Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_mspan_inuse_bytes\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545596711
Jul 17 03:14:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:33.430Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_info\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_endpoint=\"prometheus-node-exporter\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545599276
Jul 17 03:14:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:33.522Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_sockets\", asset=\"twa-tsgx05\", asset_id=\"99\", container=\"prometheus-server\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545592669
Jul 17 03:14:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:33.592Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_created\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"configmap-reloader\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545599894
Jul 17 03:14:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:33.600Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"74\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545600605
Jul 17 03:14:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:33.608Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_status_active\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", job_name=\"helm-install-prometheus\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545599894
Jul 17 03:14:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:33.625Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"85\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545600605
Jul 17 03:14:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:33.688Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_edac_csrow_correctable_errors_total\", asset=\"twa-tsgx05\", asset_id=\"99\", controller=\"0\", csrow=\"unknown\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545600605
Jul 17 03:14:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:33.719Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db174\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545609260
Jul 17 03:14:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:33.734Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_memory_max_bytes\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545609260
Jul 17 03:14:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:33.753Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"27\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545600605
Jul 17 03:14:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:33.769Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-34fd0e1b399c88f828180af36a9887dc42a1f82a23c464e843f545c11049bcd9.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"scope\"}" timestamp=1689545607691
Jul 17 03:14:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:33.827Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_bytes_total\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"cadvisor\", interface=\"ens1f0np0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545597528
Jul 17 03:14:39 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:39.659Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_gc_duration_seconds_sum\", asset=\"msh-kvm-cm001\", asset_id=\"3\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"CM-Branch-Redis-msh-kvm-cm001\", endpoint_id=\"319\", host=\"192.168.20.157\", module=\"CM-Branch-Redis\", module_id=\"73\", nz_agent_id=\"3\", parent_asset=\"msh-dellr740-pve001\", parent_asset_id=\"1\", project=\"Central-Management\", project_id=\"5\"}" timestamp=1689545618579
Jul 17 03:14:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:40.408Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dracut-shutdown.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"oneshot\", type=\"service\"}" timestamp=1689545615764
Jul 17 03:14:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:40.447Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sysstat-summary.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"oneshot\", type=\"service\"}" timestamp=1689545615764
Jul 17 03:14:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:40.622Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_labels\", asset=\"twa-tsgx04\", asset_id=\"97\", configmap=\"local-path-config\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545620034
Jul 17 03:14:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:40.849Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_vl15_dropped_total\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_10\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545611647
Jul 17 03:14:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:40.925Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_persistentvolume_status_phase\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", persistentvolume=\"pvc-fbc37096-afd3-4413-8b42-379927509ce5\", phase=\"Failed\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545620034
Jul 17 03:14:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:40.954Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_restart_policy\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-kube-state-metrics-6b768c7bf8-bvnqp\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", type=\"Always\", uid=\"e2984202-bb14-4f10-9696-64ecdaa450dd\"}" timestamp=1689545620034
Jul 17 03:14:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:41.063Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_cpu_period\", asset=\"twa-tsgx04\", asset_id=\"97\", container=\"kube-multus\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545608797
Jul 17 03:14:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:41.095Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_commands_failed_calls_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cmd=\"expire\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545613827
Jul 17 03:14:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:41.119Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"75\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545611647
Jul 17 03:14:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:41.292Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_PauseTimeWithGc_95th_percentile\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Server\"}" timestamp=1689545619757
Jul 17 03:14:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:41.311Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_incrementCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", namespace=\"hbase\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"1588230740\", table=\"meta\", type=\"data\"}" timestamp=1689545619757
Jul 17 03:14:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:41.505Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_receive_remote_physical_errors_total\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_10\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545611647
Jul 17 03:14:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:41.592Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_max_fds\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"exporter-systemd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689545615764
Jul 17 03:14:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:41.758Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dracut-pre-mount.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689545615764
Jul 17 03:14:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:41.777Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_flushesQueuedCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", namespace=\"tsg_galaxy\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"229f9f3fb448635e4ef878f09fcc95da\", table=\"relation_account_framedip\", type=\"data\"}" timestamp=1689545619757
Jul 17 03:14:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:41.886Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_errs_total\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens8f1np1\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545611647
Jul 17 03:14:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:41.982Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_numFilesCompactedCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"rtp_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"eb15417d10df48e203bcf6b7f0530100\", table=\"index_partfile_rtp_hos_bucket\", type=\"index\"}" timestamp=1689545619757
Jul 17 03:14:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:42.095Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_MEMORY_WAIT_TIME\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"phoenix\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545619757
Jul 17 03:14:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:42.197Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_hbase_table_meta_metric_splitTime_mean\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545619757
Jul 17 03:14:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:42.217Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_memStoreSize\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", namespace=\"dos\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"89022c758cf27cca0a6f1b254f97043b\", table=\"ddos_traffic_baselines\", type=\"data\"}" timestamp=1689545619757
Jul 17 03:14:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:42.238Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_up_changes_total\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"eno33np0\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545611647
Jul 17 03:14:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:42.239Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_LOG_metric_tableReadQueryPerSecond_15min_rate\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545619757
Jul 17 03:14:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:42.277Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_checkMutatePassedCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Server\"}" timestamp=1689545619757
Jul 17 03:14:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:42.336Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_fifo_total\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens6f0np0\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545611647
Jul 17 03:14:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:42.593Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"systemd-exporter-7f549d9485-7kvgn\", project=\"TSG-OS\", project_id=\"7\", reason=\"NodeLost\", service=\"prometheus-kube-state-metrics\", uid=\"27368c72-efff-405c-9fe0-1a20af809631\"}" timestamp=1689545615154
Jul 17 03:14:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:42.632Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_created\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"prometheus-server-6ff784d758\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545615154
Jul 17 03:14:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:42.633Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_storeCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"04f9e2d9401ea405fe3edb2a67813afe\", table=\"troubleshooting_hos_bucket\", type=\"data\"}" timestamp=1689545619757
Jul 17 03:14:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:42.673Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", cpu=\"94\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545616242
Jul 17 03:14:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:42.770Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_address_available\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_endpoint=\"prometheus-pushgateway\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545615154
Jul 17 03:14:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:42.790Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled_time\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-tsg-traffic-engine-numa-0-mmwxx\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"ae5d5bd3-3f8a-4eac-9409-8356a4aef052\"}" timestamp=1689545615154
Jul 17 03:14:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:42.889Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-86bb15d138648e6e0a4cd349b44db6162a88cd78ddf7e2e89e08daf7cbad6fe3.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"scope\"}" timestamp=1689545620671
Jul 17 03:14:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:42.925Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-podd20d4034_164e_4450_a851_056ee84ef97e.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"slice\"}" timestamp=1689545620671
Jul 17 03:14:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:43.007Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_reads_merged_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/run/user/1001\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545612475
Jul 17 03:14:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:43.151Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-update-utmp.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"oneshot\", type=\"service\"}" timestamp=1689545620671
Jul 17 03:14:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:43.201Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_created\", asset=\"pcap-tsgx01\", asset_id=\"71\", configmap=\"local-path-config\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545615154
Jul 17 03:14:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:43.235Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", cpu=\"37\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545616242
Jul 17 03:14:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:43.248Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_drop_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"virtio_dign_c\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545616242
Jul 17 03:14:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:43.306Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"tsg-clixon-restconf.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689545620671
Jul 17 03:14:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:43.345Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Gohangout-msh-dellr740-dt002\", endpoint_id=\"167\", module=\"DC-Gohangout\", module_id=\"35\", name=\"28\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", type=\"Processor\"}" timestamp=1689545622628
Jul 17 03:14:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:43.508Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_state_started\", asset=\"pcap-tsgx01\", asset_id=\"71\", container=\"prometheus-alertmanager-configmap-reload\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-alertmanager-6df7f457b6-m8cl7\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"98daa3ea-45f0-41c2-be94-512fb3e2a016\"}" timestamp=1689545615154
Jul 17 03:14:48 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:48.553Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_3_metric_appendTime_median\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545627438
Jul 17 03:14:48 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:48.588Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_tsg_galaxy_table_relation_account_framedip_metric_putBatchTime_75th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545627438
Jul 17 03:14:48 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:48.628Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_storeFileSize\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"rtp_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"19bdc5505a765d5acc400a971653a326\", table=\"index_time_rtp_hos_bucket\", type=\"index\"}" timestamp=1689545627438
Jul 17 03:14:48 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:48.632Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_4_metric_majorCompactionOutputFileCount_99th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545627438
Jul 17 03:14:48 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:48.641Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_maxCompactionQueueSize\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"firewall_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"e4331eab37a02574cfac02404cf8f4bf\", table=\"index_filename_firewall_hos_bucket\", type=\"index\"}" timestamp=1689545627438
Jul 17 03:14:48 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:48.665Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_hbase_table_rsgroup_metric_majorCompactionInputFileCount_mean\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545627438
Jul 17 03:14:48 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:48.981Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_1_metric_majorCompactionInputFileCount_75th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545627438
Jul 17 03:14:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:49.107Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_4_metric_putBatchTime_median\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545627438
Jul 17 03:14:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:49.144Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_2_metric_majorCompactionInputFileCount_25th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545627438
Jul 17 03:14:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:49.238Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_194_2_metric_majorCompactionTime_99_9th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545627438
Jul 17 03:14:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:49.295Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_194_1_metric_compactionOutputFileCount_99th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545627438
Jul 17 03:14:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:49.315Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_session_record_hos_bucket_metric_deleteBatchTime_median\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545627438
Jul 17 03:14:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:49.317Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_193_1_metric_putBatchTime_90th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545627438
Jul 17 03:14:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:49.457Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_readRequestCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"default\", project=\"OLAP\", project_id=\"3\", region=\"b098cb0e854858ab26062fd2fa972230\", table=\"index_partfile_default\", type=\"index\"}" timestamp=1689545627438
Jul 17 03:14:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:49.492Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_info\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", job_name=\"helm-install-traefik\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545619668
Jul 17 03:14:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:49.505Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_ready_replicas\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"prometheus-server-6ff784d758\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545619668
Jul 17 03:14:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:49.521Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_hwmon_temp_celsius\", asset=\"twa-tsgx10\", asset_id=\"109\", chip=\"pci0000:00_0000:00:19_3\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", sensor=\"temp9\", service=\"prometheus-node-exporter\"}" timestamp=1689545621797
Jul 17 03:14:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:49.547Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_scanCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"firewall_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"9c59c1eb488a29ea5a3b4aee963a09b8\", table=\"index_filename_firewall_hos_bucket\", type=\"index\"}" timestamp=1689545627438
Jul 17 03:14:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:49.558Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_SEQUENCE_metric_incrementTime_95th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545627438
Jul 17 03:14:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:49.573Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_3_metric_tableWriteQueryPerSecond_mean_rate\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545627438
Jul 17 03:14:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:49.687Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_memory_Percpu_bytes\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545621797
Jul 17 03:14:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:49.705Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_queue_length\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth43988c23\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545621797
Jul 17 03:14:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:49.707Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_scanCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"c4c7ee714d7e524ae4636d78724f09fa\", table=\"rtp_hos_bucket\", type=\"data\"}" timestamp=1689545627438
Jul 17 03:14:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:49.723Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_3_metric_averageRegionSize\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545627438
Jul 17 03:14:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:49.776Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_resource_limits\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"kube-multus\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", node=\"localhost\", nz_agent_id=\"3\", pod=\"kube-multus-ds-p9mt7\", project=\"TSG-OS\", project_id=\"7\", resource=\"cpu\", service=\"prometheus-kube-state-metrics\", uid=\"f930767b-550c-482d-9c10-4ec670104c56\", unit=\"core\"}" timestamp=1689545619668
Jul 17 03:14:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:49.806Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx10\", asset_id=\"109\", cpu=\"61\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545621797
Jul 17 03:14:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:49.954Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_errors_total\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", interface=\"eno34np1\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545618722
Jul 17 03:14:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:49.968Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-random-seed.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689545621057
Jul 17 03:14:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:49.975Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"twa-tsgx10\", asset_id=\"109\", condition=\"Available\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"tsg-traffic-engine-numa-1-firewall\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"false\"}" timestamp=1689545619668
Jul 17 03:14:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:49.996Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_bytes_total\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"eno1\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545621797
Jul 17 03:14:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:50.027Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"configmap-reloader-6c55bbc49f-wbmzc\", project=\"TSG-OS\", project_id=\"7\", reason=\"UnexpectedAdmissionError\", service=\"prometheus-kube-state-metrics\", uid=\"fd984c25-4c10-40eb-a03c-2c7c403cf652\"}" timestamp=1689545619668
Jul 17 03:14:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:50.059Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_bonding_active\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", master=\"bond0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545621797
Jul 17 03:14:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:50.259Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_node_status_condition\", asset=\"twa-tsgx08\", asset_id=\"105\", condition=\"MemoryPressure\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", node=\"localhost\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"unknown\"}" timestamp=1689545621568
Jul 17 03:14:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:50.478Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-1b9e7e11\\\\x2d9912\\\\x2d45e0\\\\x2d831b\\\\x2d8143d0877131-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2dtckrv.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689545621057
Jul 17 03:14:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:50.505Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"nfs-server.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"service\"}" timestamp=1689545621057
Jul 17 03:14:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:50.535Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_info\", address=\"a2:ec:78:4a:dd:78\", asset=\"twa-tsgx10\", asset_id=\"109\", broadcast=\"ff:ff:ff:ff:ff:ff\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethe417278f\", duplex=\"full\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operstate=\"up\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545621797
Jul 17 03:14:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:50.673Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"overlay\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-fce574c0ac2af3cd98690d5a9609653ea758628c423894910876347b94f90bee-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689545626554
Jul 17 03:14:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:50.697Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-pod59e0fcfb_c5b9_4bb0_aefc_a15fed75746e.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"slice\"}" timestamp=1689545626554
Jul 17 03:14:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:50.806Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx08\", asset_id=\"105\", container=\"firewall\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Sync\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689545625458
Jul 17 03:14:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:50.888Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_metadata_resource_version\", asset=\"twa-tsgx08\", asset_id=\"105\", configmap=\"sapp-tsg-traffic-engine-numa-1\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545621568
Jul 17 03:14:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:51.177Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"114\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545621797
Jul 17 03:14:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:51.224Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethe417278f\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545621797
Jul 17 03:14:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:51.242Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_terminated\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"helm\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-traefik-crd-h4dtp\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"440a9824-0943-4c19-88b9-06b635f437bf\"}" timestamp=1689545619668
Jul 17 03:14:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:51.299Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", cpu=\"10\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-olap003\", endpoint_id=\"235\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"yarn_resourcemanager,zookeeper,nacos,kafka,spark,druid_data,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545629713
Jul 17 03:14:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:51.370Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_netstat_Udp_SndbufErrors\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545621797
Jul 17 03:14:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:51.507Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545626709
Jul 17 03:14:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:51.531Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"119\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545626709
Jul 17 03:14:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:51.794Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_protocol_type\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"em3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-olap003\", endpoint_id=\"235\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"yarn_resourcemanager,zookeeper,nacos,kafka,spark,druid_data,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545629713
Jul 17 03:14:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:51.905Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-bond0.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689545626554
Jul 17 03:14:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:52.482Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_flags\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"bond0\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545629337
Jul 17 03:14:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:52.534Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_up\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth561c9aba\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545629337
Jul 17 03:14:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:52.555Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", phase=\"Succeeded\", pod=\"cm-policy-local-cache-redis-master-69967b5879-mfs5z\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"9b559a6c-7e7e-492f-aeb3-f47a8be58f12\"}" timestamp=1689545624478
Jul 17 03:14:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:52.758Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_restart_policy\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"svclb-traefik-644fb0c0-7cq8s\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", type=\"Always\", uid=\"ea6cc882-864f-480f-b3a6-4af3bfdaa023\"}" timestamp=1689545624478
Jul 17 03:14:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:52.920Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_bytes_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"cadvisor\", interface=\"ens6f0np0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545627675
Jul 17 03:14:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:52.956Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"tmpfs\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/data/var/lib/kubelet/pods/f2baf82c-b91b-46ec-8a59-8f55f1b86285/volumes/kubernetes.io~projected/kube-api-access-d8284\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545629337
Jul 17 03:14:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:52.973Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db10\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545619367
Jul 17 03:14:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:52.982Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_netstat_Udp_RcvbufErrors\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545629337
Jul 17 03:14:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:52.987Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-netns-cni\\\\x2d33d40af5\\\\x2dfd1d\\\\x2df2a5\\\\x2d5e82\\\\x2d8674c384cbe1.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689545625496
Jul 17 03:14:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:53.003Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_virtual_memory_max_bytes\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-journald.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689545625496
Jul 17 03:14:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:53.044Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_owner\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", owner_is_controller=\"true\", owner_kind=\"Deployment\", owner_name=\"local-path-provisioner\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"local-path-provisioner-957fdf8bc\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545624478
Jul 17 03:14:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:53.123Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_address_assign_type\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens8f1np1\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545628328
Jul 17 03:14:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:53.191Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_packets_total\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"cadvisor\", interface=\"ens3f1np1\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545615878
Jul 17 03:14:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:53.549Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"tsg_master_fail_log_s\", app_name=\"tsg_master\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"traffic-engine\", line_name=\"GTPC-RECORD\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", table_name=\"sendlog\", vsysId=\"1\"}" timestamp=1689545627622
Jul 17 03:14:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:53.557Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db108\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545619367
Jul 17 03:14:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:53.856Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", cpu=\"51\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545629337
Jul 17 03:14:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:53.866Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failcnt\", asset=\"msh-tsgx12\", asset_id=\"113\", container=\"ipmi-exporter\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545602960
Jul 17 03:14:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:53.905Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", cpu=\"26\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545629337
Jul 17 03:14:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:53.930Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_resource_limits\", asset=\"msh-tsgx12\", asset_id=\"113\", container=\"kube-multus\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", node=\"localhost\", nz_agent_id=\"3\", pod=\"kube-multus-ds-87zqt\", project=\"TSG-OS\", project_id=\"7\", resource=\"memory\", service=\"prometheus-kube-state-metrics\", uid=\"8b2e9710-4fd8-4c6f-8e5f-798d330e80a6\", unit=\"byte\"}" timestamp=1689545628663
Jul 17 03:14:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:54.028Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_terminated_reason\", asset=\"msh-tsgx08\", asset_id=\"65\", container=\"helm\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-ipmi-exporter-c8lpk\", project=\"TSG-OS\", project_id=\"7\", reason=\"Completed\", service=\"prometheus-kube-state-metrics\", uid=\"67e73254-db7c-47ad-9003-f659f9d977d7\"}" timestamp=1689545628950
Jul 17 03:14:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:54.030Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_link_error_recovery_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"mlx5_1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545619409
Jul 17 03:14:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:54.060Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"2\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545629337
Jul 17 03:14:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:54.140Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_owner\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", owner_is_controller=\"true\", owner_kind=\"ReplicaSet\", owner_name=\"coredns-749bdddcdf\", pod=\"coredns-749bdddcdf-lzwfk\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"a715b3b6-710d-4f95-8f01-a1d211f1bef3\"}" timestamp=1689545628950
Jul 17 03:14:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:54.160Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_colls_total\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"vetha1fbb3af\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545628328
Jul 17 03:14:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:54.247Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"smartd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"notify\", type=\"service\"}" timestamp=1689545625496
Jul 17 03:14:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:54.298Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_namespace_created\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545628950
Jul 17 03:14:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:54.319Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Failed\", pod=\"helm-install-tsg-traffic-engine-numa-1-glkhp\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"8ec229fc-9306-4096-93ee-079996a02e4e\"}" timestamp=1689545628950
Jul 17 03:14:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:54.380Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Udp_Link_S2C\", app_name=\"sapp4\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", vsysId=\"1\"}" timestamp=1689545627223
Jul 17 03:14:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:54.424Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-sda4.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689545625496
Jul 17 03:14:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:54.474Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled_time\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"configmap-reloader-6c55bbc49f-tmsl2\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"3af07046-1fef-40d4-a68b-5b51a863820e\"}" timestamp=1689545628663
Jul 17 03:14:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:54.491Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_colls_total\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"lo\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545623391
Jul 17 03:14:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:54.767Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx12\", asset_id=\"113\", cpu=\"88\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545623391
Jul 17 03:14:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:54.837Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_last_seen\", asset=\"msh-tsgx12\", asset_id=\"113\", container=\"prometheus-node-exporter\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545620616
Jul 17 03:14:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:54.854Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_bytes_total\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"cadvisor\", interface=\"eno2\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545617558
Jul 17 03:14:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:54.976Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", phase=\"Succeeded\", pod=\"mrzcpd-exporter-5cf7f967cc-2f76c\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"21526a39-5626-4c13-bf88-da6b44f8743f\"}" timestamp=1689545628663
Jul 17 03:14:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:55.005Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_labels\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", job_name=\"helm-install-traefik-crd\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545628663
Jul 17 03:14:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:55.006Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cpu=\"38\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545619409
Jul 17 03:14:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:55.053Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_errs_total\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"eno1\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545623391
Jul 17 03:14:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:55.186Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"promhttp_metric_handler_errors_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cause=\"encoding\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545619409
Jul 17 03:14:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:55.206Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2dpartuuid-4fcada64\\\\x2db398\\\\x2d48b0\\\\x2daa98\\\\x2d14b62c216940.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689545620884
Jul 17 03:14:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:55.353Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2dpath-pci\\\\x2d0000:81:00.0\\\\x2dscsi\\\\x2d0:2:0:0\\\\x2dpart2.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689545620884
Jul 17 03:14:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:55.380Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"mdmonitor.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"service\"}" timestamp=1689545631179
Jul 17 03:14:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:55.501Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_limit_bytes\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda5\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545625980
Jul 17 03:14:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:55.540Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_reads_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda2\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545625980
Jul 17 03:14:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:56.309Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-12893bbc4b1f51657a188cea56ff12e411716f2f83f5ce99e8acac1b22cce414.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"scope\"}" timestamp=1689545623372
Jul 17 03:14:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:56.759Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-9e2e6b5d72a23d815ec0d93e2f7feddba0689516bd71f9de8d645a40d2a63b23.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"scope\"}" timestamp=1689545623372
Jul 17 03:14:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:56.917Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_system_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", container=\"telegraf\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689545629490
Jul 17 03:14:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:56.946Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_reads_bytes_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", container=\"dign-client\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545615930
Jul 17 03:14:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:56.947Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", container=\"dign-client\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Total\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545615930
Jul 17 03:14:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:56.963Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-5ec873a9d62a4ee1b43ea962537b5563eaab2d3c39cb1e52bd946e9dc0405396-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689545623372
Jul 17 03:14:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:57.017Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:80-0000:80:01.1-0000:81:00.0-host0-target0:2:0-0:2:0:0-block-sda-sda5.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689545623372
Jul 17 03:14:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:57.270Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"clickhouse_os_user_time_cpu31\", asset=\"msh-dellr740-olap001\", asset_id=\"11\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Clickhouse-msh-dellr740-olap001\", endpoint_id=\"71\", module=\"NC-Clickhouse\", module_id=\"29\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545635798
Jul 17 03:14:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:57.328Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_cpu_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"atd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689545623372
Jul 17 03:14:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:57.894Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-ttyS3.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689545623372
Jul 17 03:14:58 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:58.017Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", container=\"dign-client\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Write\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545615930
Jul 17 03:14:58 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:58.073Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_bytes_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens4f0np0\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545625883
Jul 17 03:14:58 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:58.091Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_last_slow_execution_duration_seconds\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545626475
Jul 17 03:14:58 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:58.179Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"15\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545625883
Jul 17 03:14:58 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:58.230Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_usage_bytes\", asset=\"pcap-tsgx03\", asset_id=\"73\", container=\"kube-state-metrics\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545620989
Jul 17 03:14:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:59.775Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545629083
Jul 17 03:14:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:59.860Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"11\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545629083
Jul 17 03:14:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:59.886Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth358d74e7\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545629083
Jul 17 03:14:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:14:59.916Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-dc0b66cf\\\\x2dac45\\\\x2d4d06\\\\x2d8dbb\\\\x2d3494485cf1d5-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2d6zkqw.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689545624879
Jul 17 03:15:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:00.530Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"49\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545629083
Jul 17 03:15:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:00.816Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_multicast_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"virtio_dign_s\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545629083
Jul 17 03:15:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:00.901Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Tcp_Link_Del\", app_name=\"sapp4\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", vsysId=\"1\"}" timestamp=1689545633479
Jul 17 03:15:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:00.904Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_working_set_bytes\", asset=\"msh-tsgx01\", asset_id=\"51\", container=\"dign-server-web\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545624489
Jul 17 03:15:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:00.918Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_errors_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"cadvisor\", interface=\"br_dign_s\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545633742
Jul 17 03:15:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:01.067Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"msh-tsgx01\", asset_id=\"51\", container=\"dign-client\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"sleeping\"}" timestamp=1689545633812
Jul 17 03:15:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:01.181Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"svclb-traefik-7cfeb2e8-ftd2t\", project=\"TSG-OS\", project_id=\"7\", reason=\"Shutdown\", service=\"prometheus-kube-state-metrics\", uid=\"fc7f8bf6-1292-4741-9587-65a83ae6dca4\"}" timestamp=1689545627425
Jul 17 03:15:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:01.453Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"overlay\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-1f468c34eec82e0187cdf3a9c004f1b027d2df35fff6dd5b231be3d64d8aaedb-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689545624879
Jul 17 03:15:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:01.467Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_persistentvolume_status_phase\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", persistentvolume=\"pvc-88b47b34-97ae-4fe8-83f7-4790bbb4c0fe\", phase=\"Released\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545627425
Jul 17 03:15:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:01.567Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_errs_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"eno2\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545629083
Jul 17 03:15:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:02.347Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_metadata_resource_version\", asset=\"msh-tsgx02\", asset_id=\"63\", configmap=\"kube-root-ca.crt\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545631412
Jul 17 03:15:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:02.360Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_link_error_recovery_total\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_0\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545638318
Jul 17 03:15:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:02.392Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"msh-tsgx02\", asset_id=\"63\", cpu=\"1\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545638318
Jul 17 03:15:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:02.847Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx02\", asset_id=\"63\", container=\"traefik\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Discard\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545621085
Jul 17 03:15:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:02.872Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"msh-tsgx02\", asset_id=\"63\", condition=\"Progressing\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"prometheus-pushgateway\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"false\"}" timestamp=1689545631412
Jul 17 03:15:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:02.978Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_start_time\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", pod=\"tsg-traffic-engine-numa-1-firewall-76f6cf59ff-9hvcf\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"4a2303d1-fff8-4c08-a906-9813964074f7\"}" timestamp=1689545631412
Jul 17 03:15:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:03.028Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx02\", asset_id=\"63\", cpu=\"23\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545638318
Jul 17 03:15:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:03.253Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"msh-tsgx02\", asset_id=\"63\", condition=\"false\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"dign-client-rqnhh\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"b03a57be-3cd5-4241-ac95-41b20a48d7ef\"}" timestamp=1689545631412
Jul 17 03:15:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:03.278Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx02\", asset_id=\"63\", cpu=\"59\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545638318
Jul 17 03:15:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:03.309Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db18\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545629269
Jul 17 03:15:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:03.373Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_ips\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", ip=\"10.10.20.162\", ip_family=\"4\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"kube-multus-ds-45cnz\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"7adc00b9-e646-4822-9ef0-1a15d34bda7e\"}" timestamp=1689545631412
Jul 17 03:15:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:03.385Z caller=dedupe.go:112 component=remote level=info remote_name=52db6f url=http://10.10.10.159:10090/nz-agent/prometheus/proxy/api/v1/write msg="Dropped sample for series that was not explicitly dropped via relabelling" ref=4436650
Jul 17 03:15:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:03.472Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_constraint_errors_received_total\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_5\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545638318
Jul 17 03:15:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:03.475Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"net_conntrack_dialer_conn_established_total\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", dialer_name=\"default\", instance=\"10.10.20.159:10090\", type=\"global\"}" timestamp=1689545642990
Jul 17 03:15:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:03.498Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_memory_HugePages_Free\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545638318
Jul 17 03:15:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:03.512Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db67\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545629269
Jul 17 03:15:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:03.581Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"msh-tsgx02\", asset_id=\"63\", cpu=\"43\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545638318
Jul 17 03:15:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:03.600Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db123\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545629269
Jul 17 03:15:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:03.679Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"overlay\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-9f81d73b0a93e0d4f5d96d727831a8d4c0364cc4791475aaa2aa54ac01e432e5-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689545628447
Jul 17 03:15:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:03.745Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"initrd-parse-etc.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689545628447
Jul 17 03:15:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:04.120Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"prometheus_http_response_size_bytes_bucket\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", handler=\"/api/v1/query\", instance=\"10.10.20.159:10090\", le=\"1e+06\", type=\"global\"}" timestamp=1689545642990
Jul 17 03:15:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:04.139Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_usage_bytes\", asset=\"msh-tsgx02\", asset_id=\"63\", container=\"lb-tcp-443\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545616323
Jul 17 03:15:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:04.331Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"msh-tsgx02\", asset_id=\"63\", container=\"local-path-provisioner\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", failure_type=\"pgmajfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"hierarchy\"}" timestamp=1689545628078
Jul 17 03:15:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:06.756Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx01\", asset_id=\"91\", container=\"dign-client\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Async\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545629217
Jul 17 03:15:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:06.776Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_spec_paused\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"metrics-server\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545641227
Jul 17 03:15:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:06.869Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_packets_total\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth0799f3ea\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545643910
Jul 17 03:15:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:06.909Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db108\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545638290
Jul 17 03:15:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:06.921Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_disk_io_time_seconds_total\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"sda\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545643910
Jul 17 03:15:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:06.993Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dracut-shutdown.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"service\"}" timestamp=1689545633582
Jul 17 03:15:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:07.102Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_multicast_total\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"em2\", endpoint=\"NC-Spark-msh-dellr740-olap003\", endpoint_id=\"13\", module=\"NC-Spark\", module_id=\"47\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545646092
Jul 17 03:15:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:07.213Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Bit_Outbound\", app_name=\"sapp4\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", vsysId=\"1\"}" timestamp=1689545637817
Jul 17 03:15:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:07.258Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:40-0000:40:01.1-0000:41:00.1-infiniband_mad-umad5.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689545633582
Jul 17 03:15:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:07.284Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sysstat-summary.timer\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"timer\"}" timestamp=1689545633582
Jul 17 03:15:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:07.300Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas_ready\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"configmap-reloader\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545641227
Jul 17 03:15:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:07.342Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_link_error_recovery_total\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545643910
Jul 17 03:15:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:07.372Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_up_changes_total\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth73a33e2a\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545643910
Jul 17 03:15:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:07.401Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_complete\", asset=\"twa-tsgx01\", asset_id=\"91\", condition=\"true\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", job_name=\"helm-install-ipmi-exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545641227
Jul 17 03:15:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:07.482Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", phase=\"Succeeded\", pod=\"cm-policy-local-cache-redis-master-69967b5879-qz5fq\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"7ce93833-4c85-4a59-be25-1ba1b6f91e91\"}" timestamp=1689545641227
Jul 17 03:15:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:07.503Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"96\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545643910
Jul 17 03:15:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:07.587Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", cpu=\"37\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Spark-msh-dellr740-olap003\", endpoint_id=\"13\", mode=\"irq\", module=\"NC-Spark\", module_id=\"47\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545646092
Jul 17 03:15:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:07.711Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_bytes_total\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"cadvisor\", interface=\"ens6f0np0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545638270
Jul 17 03:15:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:07.728Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_cpu_period\", asset=\"twa-tsgx01\", asset_id=\"91\", container=\"dign-server-web\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545642966
Jul 17 03:15:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:07.741Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_allocator_frag_ratio\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545638290
Jul 17 03:15:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:07.748Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_init_container_status_terminated_reason\", asset=\"twa-tsgx01\", asset_id=\"91\", container=\"firewall-init\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", pod=\"tsg-traffic-engine-numa-1-firewall-7f6cb768db-2lm6c\", project=\"TSG-OS\", project_id=\"7\", reason=\"Completed\", service=\"prometheus-kube-state-metrics\", uid=\"9b7a7ee8-3d4c-4d13-b2bb-d0637b89d0c8\"}" timestamp=1689545641227
Jul 17 03:15:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:07.760Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"irqbalance.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"service\"}" timestamp=1689545633582
Jul 17 03:15:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:07.770Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cpu=\"8\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545643910
Jul 17 03:15:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:07.904Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"twa-tsgx01\", asset_id=\"91\", condition=\"unknown\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-traefik-crd-z94x2\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"822cc042-6602-4322-90e4-fde792feba93\"}" timestamp=1689545641227
Jul 17 03:15:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:07.916Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db0\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545638290
Jul 17 03:15:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:07.941Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_multicast_total\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"virtio_dign_s\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545643910
Jul 17 03:15:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:08.112Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_errs_total\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"eno2\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545643910
Jul 17 03:15:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:08.378Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"twa-tsgx01\", asset_id=\"91\", condition=\"unknown\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"dign-client-l54s6\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"37411639-54a5-4d19-bf38-3ad466e16fc4\"}" timestamp=1689545641227
Jul 17 03:15:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:08.392Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cpu=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545643910
Jul 17 03:15:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:08.402Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-sda1.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689545633582
Jul 17 03:15:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:08.942Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_drop_total\", asset=\"pcap-tsgx06\", asset_id=\"81\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"vethdbe33966\", endpoint=\"TSG-OS-pcap-tsgx06\", endpoint_id=\"475\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545636740
Jul 17 03:15:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:10.535Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_getCount\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"rtp_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"f687909ec9efaa523dad1a1f63f40ef4\", table=\"index_partfile_rtp_hos_bucket\", type=\"index\"}" timestamp=1689545648804
Jul 17 03:15:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:10.545Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_avgStoreFileAge\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"default\", project=\"OLAP\", project_id=\"3\", region=\"5ea584a7f079803450378cad2c7006a0\", table=\"index_filename_default\", type=\"index\"}" timestamp=1689545648804
Jul 17 03:15:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:10.548Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_RenewalFailuresTotal\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", module=\"DC-HBase\", module_id=\"3\", name=\"UgiMetrics\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545648804
Jul 17 03:15:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:10.820Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_compactionsCompletedCount\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"rtp_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"d64e085bec7dcaecff2f6350cdf00ff6\", table=\"index_time_rtp_hos_bucket\", type=\"index\"}" timestamp=1689545648804
Jul 17 03:15:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:10.923Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_numBytesCompactedCount\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"firewall_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"ebb46fb579a2b3ca1735fe995db38a0a\", table=\"index_filename_firewall_hos_bucket\", type=\"index\"}" timestamp=1689545648804
Jul 17 03:15:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:10.980Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_created\", asset=\"twa-tsgx11\", asset_id=\"119\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx11\", endpoint_id=\"461\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"ipmi-exporter\", uid=\"e99f0aee-bfed-4061-bcd8-bbad1cf3c78c\"}" timestamp=1689545641492
Jul 17 03:15:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:11.323Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_tsg_galaxy_table_relation_framedip_account_metric_deleteTime_75th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545650147
Jul 17 03:15:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:11.335Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_flushesQueuedCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"proxy_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"513c53b10d036c82752eb4ff5811ee84\", table=\"index_partfile_proxy_hos_bucket\", type=\"index\"}" timestamp=1689545650147
Jul 17 03:15:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:11.452Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_193_1_metric_compactionOutputSize_95th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545650147
Jul 17 03:15:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:11.595Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_194_2_metric_majorCompactionOutputFileCount_mean\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545650147
Jul 17 03:15:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:11.617Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_194_4_metric_compactionTime_mean\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545650147
Jul 17 03:15:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:11.678Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_TASK_metric_putBatchTime_min\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545650147
Jul 17 03:15:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:11.718Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_default_metric_deleteTime_min\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545650147
Jul 17 03:15:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:11.724Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_4_metric_incrementTime_95th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545650147
Jul 17 03:15:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:11.739Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_3_metric_incrementTime_95th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545650147
Jul 17 03:15:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:11.757Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_default_metric_compactionTime_median\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545650147
Jul 17 03:15:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:11.784Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_tsg_table_report_result_metric_putBatchTime_mean\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545650147
Jul 17 03:15:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:11.910Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_maxFlushQueueSize\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"default\", project=\"OLAP\", project_id=\"3\", region=\"eae1de6a9b126aa98cddbc256bd9d749\", table=\"index_time_default\", type=\"index\"}" timestamp=1689545650147
Jul 17 03:15:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:11.955Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_tsg_table_report_result_metric_appendTime_median\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545650147
Jul 17 03:15:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:11.972Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_194_1_metric_appendTime_median\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545650147
Jul 17 03:15:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:11.989Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_session_record_hos_bucket_metric_flushTime_99th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545650147
Jul 17 03:15:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:12.011Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_1_metric_incrementTime_median\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545650147
Jul 17 03:15:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:12.127Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_avgStoreFileAge\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"rtp_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"975a4eaf054d84bc134e1d1c63685744\", table=\"index_filename_rtp_hos_bucket\", type=\"index\"}" timestamp=1689545650147
Jul 17 03:15:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:12.134Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_session_record_hos_bucket_metric_scanTime_median\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545650147
Jul 17 03:15:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:12.150Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_maxStoreFileAge\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"firewall_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"0b82285c269281f43c5e890a18d289b9\", table=\"index_filename_firewall_hos_bucket\", type=\"index\"}" timestamp=1689545650147
Jul 17 03:15:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:12.215Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_3_metric_putTime_mean\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545650147
Jul 17 03:15:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:12.223Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_getCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"707c60fcbf33b487dea94876da6b9b4d\", table=\"default\", type=\"data\"}" timestamp=1689545650147
Jul 17 03:15:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:12.268Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_maxStoreFileAge\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"default\", project=\"OLAP\", project_id=\"3\", region=\"407d7190eb2c126babdaa4f2a96fab6b\", table=\"index_filename_default\", type=\"index\"}" timestamp=1689545650147
Jul 17 03:15:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:12.285Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_193_1_metric_incrementTime_98th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545650147
Jul 17 03:15:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:12.324Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_4_metric_majorCompactionOutputFileCount_num_ops\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545650147
Jul 17 03:15:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:12.344Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_tsg_galaxy_table_gtpc_knowledge_base_metric_deleteBatchTime_min\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545650147
Jul 17 03:15:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:12.588Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_netstat_IpExt_OutOctets\", asset=\"twa-dellr740-dt002\", asset_id=\"39\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"OLAP-Node-Exporter-twa-dellr740-dt002\", endpoint_id=\"259\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545652055
Jul 17 03:15:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:12.712Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_4_metric_scanSize_75th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545650147
Jul 17 03:15:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:12.713Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_STATS_metric_deleteBatchTime_min\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545650147
Jul 17 03:15:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:12.894Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_194_3_metric_compactionOutputSize_99th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545650147
Jul 17 03:15:13 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:13.089Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_putCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"proxy_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"2215ad860c53cf83696b749e8a270a13\", table=\"index_time_proxy_hos_bucket\", type=\"index\"}" timestamp=1689545650147
Jul 17 03:15:13 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:13.114Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_filteredReadRequestCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"d8d406de01d305b84926bfbbb7ac5538\", table=\"rtp_hos_bucket\", type=\"data\"}" timestamp=1689545650147
Jul 17 03:15:13 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:13.303Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_numReferenceFiles\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"00dc5a251addf7b4aaed51561226ada8\", table=\"proxy_hos_bucket\", type=\"data\"}" timestamp=1689545650147
Jul 17 03:15:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:15.248Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_name_assign_type\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth48e28e0d\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545641230
Jul 17 03:15:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:15.258Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_speed_bytes\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"eno2\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545641230
Jul 17 03:15:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:15.470Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_speed_bytes\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens3f0np0\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545641230
Jul 17 03:15:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:16.010Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"overlay\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-5d8ba54f071fbeea504df47b857740cc06184da019235853e66b2dc0d22ef817-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689545647046
Jul 17 03:15:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:16.169Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_spec_replicas\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"dashboard-metrics-scraper\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545648076
Jul 17 03:15:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:16.195Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-554896d488f4e8a2f24d4292c4f8e7737fee9bfa4a62e85997c3eeb92497fabb.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"scope\"}" timestamp=1689545647046
Jul 17 03:15:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:16.903Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"23\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545641230
Jul 17 03:15:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:19.633Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_name_assign_type\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens2f0np0\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545657612
Jul 17 03:15:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:19.639Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"84\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545657612
Jul 17 03:15:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:19.656Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"nfs-server.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"service\"}" timestamp=1689545644356
Jul 17 03:15:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:19.726Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_state_started\", asset=\"twa-tsgx06\", asset_id=\"101\", container=\"prometheus-server-configmap-reload\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-server-6ff784d758-zzlsk\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"9041122f-12ad-44d4-a492-63eb077aa8d8\"}" timestamp=1689545653949
Jul 17 03:15:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:19.763Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"3\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545657612
Jul 17 03:15:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:20.057Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_packets_total\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"br_dign_s\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545657612
Jul 17 03:15:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:20.060Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethac193cfe\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545657612
Jul 17 03:15:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:20.512Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"113\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545657612
Jul 17 03:15:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:20.556Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545657612
Jul 17 03:15:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:20.585Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"virtio_dign_c\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545657612
Jul 17 03:15:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:20.601Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_name_assign_type\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth3177b6ed\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545657612
Jul 17 03:15:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:20.763Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", container=\"dign-server-web\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"cadvisor\", failure_type=\"pgfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"hierarchy\"}" timestamp=1689545644687
Jul 17 03:15:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:20.784Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"ipmi_up\", asset=\"pcap-tsgx10\", asset_id=\"89\", collector=\"bmc\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"ipmi-exporter\"}" timestamp=1689545648448
Jul 17 03:15:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:20.785Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_usage_bytes\", asset=\"pcap-tsgx10\", asset_id=\"89\", container=\"firewall\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689545651242
Jul 17 03:15:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:20.802Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_transmit_wait_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"mlx5_11\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545648263
Jul 17 03:15:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:20.823Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_spec_type\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"ipmi-exporter\", type=\"ClusterIP\", uid=\"49c5b98d-5032-456b-8431-d4a339a1f041\"}" timestamp=1689545653828
Jul 17 03:15:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:21.064Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_fifo_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"eno35np2\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545648263
Jul 17 03:15:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:21.163Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"34\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545657612
Jul 17 03:15:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:21.232Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", container=\"exporter-merger\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"uninterruptible\"}" timestamp=1689545649196
Jul 17 03:15:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:21.250Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"tmpfs\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/data/var/lib/kubelet/pods/a74e5213-995f-4251-a2d7-39614c7890a8/volumes/kubernetes.io~projected/kube-api-access-25q2k\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545657612
Jul 17 03:15:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:21.285Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_address_assign_type\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens8f1np1\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545657612
Jul 17 03:15:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:21.580Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db119\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545651998
Jul 17 03:15:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:21.677Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_edac_csrow_correctable_errors_total\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", controller=\"1\", csrow=\"0\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"OLAP-Node-Exporter-twa-dellr740-dt001\", endpoint_id=\"257\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545661408
Jul 17 03:15:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:22.203Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"twa-tsgx06\", asset_id=\"101\", condition=\"Progressing\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"configmap-reloader\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"true\"}" timestamp=1689545653949
Jul 17 03:15:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:22.217Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"13\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545657612
Jul 17 03:15:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:22.402Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link_mode\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth44c0947a\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545648263
Jul 17 03:15:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:22.449Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_info\", address=\"08:c0:eb:53:c8:d5\", asset=\"pcap-tsgx10\", asset_id=\"89\", broadcast=\"ff:ff:ff:ff:ff:ff\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens3f1np1\", duplex=\"unknown\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operstate=\"down\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545648263
Jul 17 03:15:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:22.481Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_node_status_capacity\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", node=\"localhost\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", resource=\"ephemeral_storage\", service=\"prometheus-kube-state-metrics\", unit=\"byte\"}" timestamp=1689545653949
Jul 17 03:15:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:22.489Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Running\", pod=\"metrics-server-54dc485875-dwq62\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"c5e80f05-95b8-4a08-9917-e24687aa238e\"}" timestamp=1689545653949
Jul 17 03:15:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:22.853Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_heap_alloc_bytes\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"OLAP-Node-Exporter-twa-dellr740-dt001\", endpoint_id=\"257\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545661408
Jul 17 03:15:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:23.081Z caller=dedupe.go:112 component=remote level=error remote_name=52db6f url=http://10.10.10.159:10090/nz-agent/prometheus/proxy/api/v1/write msg="non-recoverable error" count=1000 exemplarCount=0 err="server returned HTTP status 400 : out of order sample"
Jul 17 03:15:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:23.417Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_annotations\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", job_name=\"helm-install-tsg-traffic-engine-numa-1\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545653828
Jul 17 03:15:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:23.509Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_gc_duration_seconds\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"OLAP-Node-Exporter-twa-dellr740-dt001\", endpoint_id=\"257\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\", quantile=\"0.75\"}" timestamp=1689545661408
Jul 17 03:15:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:23.632Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"msh-tsgx09\", asset_id=\"55\", cpu=\"17\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545654790
Jul 17 03:15:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:23.673Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"OLAP-Node-Exporter-twa-dellr740-dt001\", endpoint_id=\"257\", module=\"OLAP-Node-Exporter\", module_id=\"51\", name=\"15\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\", type=\"Processor\"}" timestamp=1689545661408
Jul 17 03:15:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:23.920Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"msh-tsgx09\", asset_id=\"55\", cpu=\"117\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545654790
Jul 17 03:15:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:23.993Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"msh-tsgx09\", asset_id=\"55\", cpu=\"9\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545654790
Jul 17 03:15:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:24.026Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-vconsole-setup.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689545658416
Jul 17 03:15:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:24.144Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-b40f3139a985682e40ec310679e88017429c5d22e99d3b0f8b73c9d6ea1009c6.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"scope\"}" timestamp=1689545651102
Jul 17 03:15:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:24.162Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db144\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545657019
Jul 17 03:15:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:24.172Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-3edc99ea9fb8cf294828bd9d36157ab408f7c2a6d759b4c3d9f6e34cd6e06acd.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"scope\"}" timestamp=1689545651102
Jul 17 03:15:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:24.207Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-077ad337df6aa6d2b1064c1f0e3036e1dfd0a8e71e3f3daf0635f5be98ec9e4c-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689545651102
Jul 17 03:15:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:24.239Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"82\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545660160
Jul 17 03:15:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:24.353Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"phy_tx_bits_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", dev=\"ens7f1np1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"mrzcpd-exporter\"}" timestamp=1689545655378
Jul 17 03:15:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:24.379Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"etc.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689545651102
Jul 17 03:15:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:24.432Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"lvm2-monitor.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689545651102
Jul 17 03:15:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:24.572Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_start_time\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"traefik-64f55bb67d-xmdbm\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"012398ff-fe58-4ee4-8731-8e61bf317bbf\"}" timestamp=1689545660068
Jul 17 03:15:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:24.733Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-ad072d0966fd4202b0ebb83f863b93e7153f7bd40b9a0a2d179e19218d15c0a2-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689545658416
Jul 17 03:15:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:24.864Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"pcap-tsgx12\", asset_id=\"117\", condition=\"false\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", pod=\"tsg-traffic-engine-numa-0-firewall-64484c6cd-h4bj8\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"692f1a13-48e0-47b7-ad46-a5b348bd402b\"}" timestamp=1689545651122
Jul 17 03:15:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:25.007Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_threads\", asset=\"msh-tsgx09\", asset_id=\"55\", container=\"ipmi-exporter\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545657961
Jul 17 03:15:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:25.029Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_init_container_status_running\", asset=\"msh-tsgx09\", asset_id=\"55\", container=\"firewall-init\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", pod=\"tsg-traffic-engine-numa-1-firewall-5f74b447d7-bwdnr\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"f2551d6b-0939-434d-ba32-2e369c56471c\"}" timestamp=1689545660068
Jul 17 03:15:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:25.167Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"30\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545660160
Jul 17 03:15:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:25.228Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"38\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545660160
Jul 17 03:15:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:25.240Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_packets_dropped_total\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", interface=\"eno34np1\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545657488
Jul 17 03:15:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:25.273Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"kubernetes-dashboard\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", failure_type=\"pgfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"hierarchy\"}" timestamp=1689545660820
Jul 17 03:15:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:25.286Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_rss\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"prometheus-alertmanager-configmap-reload\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545651446
Jul 17 03:15:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:25.298Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_file_descriptors\", asset=\"msh-tsgx09\", asset_id=\"55\", container=\"prometheus-node-exporter\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545646711
Jul 17 03:15:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:25.305Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_labels\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", job_name=\"helm-install-prometheus\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545651122
Jul 17 03:15:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:25.341Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_packets_total\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"virtio_dign_s\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545654790
Jul 17 03:15:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:25.348Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_reservation_limit_bytes\", asset=\"msh-tsgx09\", asset_id=\"55\", container=\"prometheus-pushgateway\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545658586
Jul 17 03:15:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:25.388Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_errs_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"vethc31b7dd7\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545660160
Jul 17 03:15:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:25.410Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_annotations\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"chart-values-ipmi-exporter\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545651122
Jul 17 03:15:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:25.436Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"pcap-tsgx12\", asset_id=\"117\", condition=\"Available\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"prometheus-pushgateway\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"true\"}" timestamp=1689545651122
Jul 17 03:15:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:25.438Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"rshim.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"service\"}" timestamp=1689545651102
Jul 17 03:15:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:25.445Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"ipmi.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"service\"}" timestamp=1689545658416
Jul 17 03:15:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:25.447Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_persistentvolume_info\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", persistentvolume=\"pvc-06f1fc97-f004-48b3-97d5-b2c7e3231e2b\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", storageclass=\"local-path\"}" timestamp=1689545651122
Jul 17 03:15:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:25.585Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"tmpfs\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/var/lib/kubelet/pods/f7af3cd7-b972-4aba-a12a-7b5c827e57eb/volumes/kubernetes.io~projected/kube-api-access-84tdc\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545660160
Jul 17 03:15:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:25.614Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_status_failed\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", job_name=\"helm-install-cm-policy-local-cache\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545651122
Jul 17 03:15:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:25.632Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_ready\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"helm\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-ipmi-exporter-wk46t\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"7a3c1563-fe1c-4a2b-b7d1-5ba87553013e\"}" timestamp=1689545651122
Jul 17 03:15:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:25.654Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-br_dign_s.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689545658416
Jul 17 03:15:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:25.660Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_fifo_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"vethfc5d9f9d\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545660160
Jul 17 03:15:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:25.749Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_bytes_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens3f0np0\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545660160
Jul 17 03:15:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:25.772Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db133\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545661795
Jul 17 03:15:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:25.774Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"7\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545660160
Jul 17 03:15:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:25.778Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_file_descriptors\", asset=\"msh-tsgx09\", asset_id=\"55\", container=\"metrics-server\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545658216
Jul 17 03:15:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:25.836Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_queue_length\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth0c6b6d72\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545654790
Jul 17 03:15:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:25.852Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-ens1f1np1.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689545658416
Jul 17 03:15:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:25.881Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kafka_server_BrokerTopicMetrics_OneMinuteRate\", asset=\"msh-dellr740-dt003\", asset_id=\"9\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-Kafka-msh-dellr740-dt003\", endpoint_id=\"97\", module=\"DC-Kafka\", module_id=\"5\", name=\"MessagesInPerSec\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", topic=\"BGP-RECORD\"}" timestamp=1689545664888
Jul 17 03:15:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:26.016Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_processes\", asset=\"twa-tsgx09\", asset_id=\"107\", container=\"telegraf\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689545654869
Jul 17 03:15:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:26.060Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx09\", asset_id=\"107\", container=\"dign-server-dns\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Discard\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545650489
Jul 17 03:15:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:26.080Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"32\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545655810
Jul 17 03:15:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:26.147Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Succeeded\", pod=\"helm-install-ipmi-exporter-wk46t\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"7a3c1563-fe1c-4a2b-b7d1-5ba87553013e\"}" timestamp=1689545651122
Jul 17 03:15:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:26.159Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas_updated\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"mrzcpd-exporter\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545660068
Jul 17 03:15:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:26.166Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dracut-pre-mount.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689545653907
Jul 17 03:15:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:26.193Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx09\", asset_id=\"55\", cpu=\"31\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545654790
Jul 17 03:15:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:26.208Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_restart_policy\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-cm-policy-local-cache-wzw8p\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", type=\"OnFailure\", uid=\"57683893-10c5-486b-89a6-dbc1429ed965\"}" timestamp=1689545658744
Jul 17 03:15:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:26.230Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"0\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545655810
Jul 17 03:15:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:26.255Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_created\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-0-firewall-d55dcdb59\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545658744
Jul 17 03:15:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:26.257Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_reservation_limit_bytes\", asset=\"msh-tsgx09\", asset_id=\"55\", container=\"dign-server-dns\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545658586
Jul 17 03:15:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:26.269Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_address_assign_type\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth9ad1694f\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545655810
Jul 17 03:15:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:26.282Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"57\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545655810
Jul 17 03:15:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:26.327Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_user_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", container=\"ipmi-exporter\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545663502
Jul 17 03:15:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:26.370Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_errors_received_total\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_11\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545654790
Jul 17 03:15:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:26.418Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_dormant\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth9df3a32a\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545654790
Jul 17 03:15:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:26.457Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"abrt-xorg.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"service\"}" timestamp=1689545653907
Jul 17 03:15:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:26.512Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_cpu_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"rdma-ndd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689545653907
Jul 17 03:15:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:26.555Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"crond.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"service\"}" timestamp=1689545653907
Jul 17 03:15:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:26.682Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Sync\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689545657065
Jul 17 03:15:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:27.020Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_device_id\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens6f1np1\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545655810
Jul 17 03:15:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:27.301Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_labels\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-server\", uid=\"5d69a0e6-abb0-4748-927e-25c8b27a7f8d\"}" timestamp=1689545651122
Jul 17 03:15:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:27.303Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_packets_dropped_total\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"cadvisor\", interface=\"ens3f0np0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545661591
Jul 17 03:15:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:27.329Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_spec_replicas\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"local-path-provisioner\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545658744
Jul 17 03:15:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:27.368Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sockets.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"target\"}" timestamp=1689545653907
Jul 17 03:15:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:27.387Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"ftx_bits_total\", app=\"sapp4-0\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"nf_0_fw\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"mrzcpd-exporter\", serviceFunctionIndex=\"0\"}" timestamp=1689545658295
Jul 17 03:15:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:27.397Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"clickhouse_os_steal_time_cpu44\", asset=\"msh-dellr740-dt003\", asset_id=\"9\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Clickhouse-msh-dellr740-dt003\", endpoint_id=\"75\", module=\"NC-Clickhouse\", module_id=\"29\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545666493
Jul 17 03:15:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:27.419Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"43\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545655810
Jul 17 03:15:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:27.434Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"8\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545662882
Jul 17 03:15:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:27.455Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_labels\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"cm-policy-local-cache-redis-master\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545651122
Jul 17 03:15:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:27.522Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_ulimits_soft\", asset=\"twa-tsgx09\", asset_id=\"107\", container=\"firewall\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", ulimit=\"max_open_files\"}" timestamp=1689545663745
Jul 17 03:15:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:27.544Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_unicast_packets_received_total\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_9\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545655810
Jul 17 03:15:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:27.601Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_swap_limit_bytes\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545663419
Jul 17 03:15:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:27.811Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_multicast_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"eno34np1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545660160
Jul 17 03:15:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:27.954Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_start_time\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-configmap-reloader-szrcq\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"99814cb0-81f7-4fa6-9f8c-54016e6c8558\"}" timestamp=1689545655682
Jul 17 03:15:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:27.971Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_multicast_packets_received_total\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_0\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545655810
Jul 17 03:15:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:28.003Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"88\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545655810
Jul 17 03:15:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:28.013Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"28\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545655810
Jul 17 03:15:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:28.039Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"coredns-647dbfccd5-sr892\", project=\"TSG-OS\", project_id=\"7\", reason=\"NodeAffinity\", service=\"prometheus-kube-state-metrics\", uid=\"ce676e45-7527-495c-945c-c9fa2e1248cd\"}" timestamp=1689545655682
Jul 17 03:15:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:28.061Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_bytes_total\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"bond0\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545655810
Jul 17 03:15:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:28.087Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"nfs-utils.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"service\"}" timestamp=1689545654114
Jul 17 03:15:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:28.213Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Discard\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545664183
Jul 17 03:15:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:28.216Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-d49e1f92ac144fe07247560cf84acd290693f407eceaa245acdd5295cccc7637-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689545653907
Jul 17 03:15:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:28.514Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_device_id\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth476a8450\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545662882
Jul 17 03:15:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:28.552Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db138\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545659446
Jul 17 03:15:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:28.653Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_mapped_file\", asset=\"twa-tsgx09\", asset_id=\"107\", container=\"kubernetes-dashboard\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545652186
Jul 17 03:15:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:28.726Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_info\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_endpoint=\"kubernetes\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545658744
Jul 17 03:15:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:28.760Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"rpcbind.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"target\"}" timestamp=1689545654114
Jul 17 03:15:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:28.970Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-quotacheck.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"oneshot\", type=\"service\"}" timestamp=1689545654114
Jul 17 03:15:29 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:29.038Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-veth30e28f20.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689545654114
Jul 17 03:15:29 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:29.064Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545655810
Jul 17 03:15:29 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:29.215Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-tsg-traffic-engine-numa-1-7rxbb\", project=\"TSG-OS\", project_id=\"7\", reason=\"NodeAffinity\", service=\"prometheus-kube-state-metrics\", uid=\"03195aa0-7d32-4be5-b57f-109823710aac\"}" timestamp=1689545655682
Jul 17 03:15:29 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:29.386Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"tsg_master_del\", app_name=\"tsg_master\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"traffic-engine\", line_name=\"compile\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", table_name=\"sync_exdata\", vsysId=\"1\"}" timestamp=1689545664090
Jul 17 03:15:29 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:29.500Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_oom_events_total\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"prometheus-server-configmap-reload\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545662803
Jul 17 03:15:29 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:29.549Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"msh-tsgx10\", asset_id=\"69\", condition=\"Available\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"metrics-server\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"false\"}" timestamp=1689545655682
Jul 17 03:15:30 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:30.130Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545662882
Jul 17 03:15:30 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:30.984Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_daemonset_status_observed_generation\", asset=\"msh-tsgx05\", asset_id=\"67\", daemonset=\"kube-multus-ds\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545664068
Jul 17 03:15:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:31.164Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_labels\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"metrics-server\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545664068
Jul 17 03:15:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:31.545Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_ready\", asset=\"twa-tsgx03\", asset_id=\"95\", container=\"firewall\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", pod=\"tsg-traffic-engine-numa-1-firewall-dc9c7db68-v2sqf\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"1df6bd2b-0fea-4549-94c2-ddadc3bbc3e4\"}" timestamp=1689545670436
Jul 17 03:15:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:31.592Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"118\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545660939
Jul 17 03:15:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:31.600Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_created\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"kubernetes\", uid=\"c0490602-8741-4474-9b02-c83bfe505e0a\"}" timestamp=1689545670436
Jul 17 03:15:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:31.600Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"7\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545659093
Jul 17 03:15:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:31.678Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"getty.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"target\"}" timestamp=1689545661015
Jul 17 03:15:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:31.796Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_id\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth6bde53db\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545659093
Jul 17 03:15:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:31.851Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_info\", address=\"7e:2a:be:cb:59:19\", asset=\"twa-tsgx03\", asset_id=\"95\", broadcast=\"ff:ff:ff:ff:ff:ff\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth2118d09a\", duplex=\"full\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operstate=\"up\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545659093
Jul 17 03:15:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:32.072Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"metrics\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", failure_type=\"pgfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"container\"}" timestamp=1689545659396
Jul 17 03:15:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:32.165Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545659093
Jul 17 03:15:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:32.179Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"vgauthd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"simple\", type=\"service\"}" timestamp=1689545666907
Jul 17 03:15:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:32.497Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Succeeded\", pod=\"local-path-provisioner-957fdf8bc-rf5f7\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"f2034e38-7d8b-4f82-8274-cb5f14eacda8\"}" timestamp=1689545659894
Jul 17 03:15:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:32.520Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_bytes_total\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", interface=\"eno33np0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545660278
Jul 17 03:15:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:32.521Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"42\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545660605
Jul 17 03:15:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:32.549Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"tmpfs\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/var/lib/kubelet/pods/e3971f33-46bb-450e-a67b-8e9e2f2ec265/volumes/kubernetes.io~projected/kube-api-access-tj29s\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545660605
Jul 17 03:15:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:32.568Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_created\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"chart-values-tsg-traffic-engine-numa-0\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545659276
Jul 17 03:15:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:32.601Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_cpu_period\", asset=\"twa-tsgx03\", asset_id=\"95\", container=\"prometheus-server-configmap-reload\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545663123
Jul 17 03:15:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:32.621Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"rc-local.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689545666907
Jul 17 03:15:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:32.623Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Succeeded\", pod=\"helm-install-prometheus-mshnr\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"6fb2bd33-bd15-4e41-9452-847ce05e8c34\"}" timestamp=1689545670436
Jul 17 03:15:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:32.645Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-veth662ec9f2.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689545666907
Jul 17 03:15:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:32.652Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"58\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545659093
Jul 17 03:15:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:32.672Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"configmap-reloader-6c55bbc49f-qmcwt\", project=\"TSG-OS\", project_id=\"7\", reason=\"NodeLost\", service=\"prometheus-kube-state-metrics\", uid=\"03b47e9e-b9a5-462d-b6ff-6e3865bd5555\"}" timestamp=1689545670436
Jul 17 03:15:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:32.689Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_drop_total\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens4f3np3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545660939
Jul 17 03:15:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:32.693Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_virtual_memory_max_bytes\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"auditd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689545666907
Jul 17 03:15:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:32.720Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_limit_bytes\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545658535
Jul 17 03:15:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:32.721Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_cpu_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-pod9a50710b_a0ce_44fe_abf8_917fc3e36a13.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"slice\"}" timestamp=1689545666907
Jul 17 03:15:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:32.751Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_address_available\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_endpoint=\"cm-policy-local-cache-redis-headless\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545670436
Jul 17 03:15:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:32.791Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"iscsi-shutdown.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"oneshot\", type=\"service\"}" timestamp=1689545662644
Jul 17 03:15:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:32.809Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_daemonset_status_number_ready\", asset=\"twa-tsgx05\", asset_id=\"99\", daemonset=\"kube-multus-ds\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545659894
Jul 17 03:15:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:32.826Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"initrd-cleanup.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"service\"}" timestamp=1689545662644
Jul 17 03:15:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:32.869Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"twa-tsgx05\", asset_id=\"99\", container=\"kubernetes-dashboard\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"iowaiting\"}" timestamp=1689545655216
Jul 17 03:15:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:32.886Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_working_set_bytes\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"prometheus-node-exporter\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545647005
Jul 17 03:15:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:32.946Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_info\", asset=\"twa-tsgx05\", asset_id=\"99\", created_by_kind=\"ReplicaSet\", created_by_name=\"dashboard-metrics-scraper-5b94897dc9\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", host_ip=\"10.10.30.165\", host_network=\"false\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", node=\"localhost\", nz_agent_id=\"3\", pod=\"dashboard-metrics-scraper-5b94897dc9-cbj7z\", pod_ip=\"10.42.0.61\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"3b7c443c-5d2b-4a7a-8d44-9283e8202f9c\"}" timestamp=1689545659894
Jul 17 03:15:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:32.947Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", phase=\"Succeeded\", pod=\"systemd-exporter-7f549d9485-xczgq\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"ef20aecf-4336-4e43-9a1d-c18770bb0139\"}" timestamp=1689545659894
Jul 17 03:15:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:33.235Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"34\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545660605
Jul 17 03:15:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:33.329Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_annotations\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"svclb-traefik-f8ab0b79-f6t4m\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"6c839a6b-757d-460b-acb3-20a2f3df359e\"}" timestamp=1689545659894
Jul 17 03:15:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:33.343Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_cpu_period\", asset=\"msh-tsgx05\", asset_id=\"67\", container=\"kube-multus\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545655836
Jul 17 03:15:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:33.554Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", fstype=\"ext4\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/data/var/lib/kubelet/pods/bdd10b0d-cc33-4a4f-9622-89ebf267e6c3/volume-subpaths/config-volume/firewall/1\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545660605
Jul 17 03:15:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:33.590Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_rss\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"dign-client\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545657378
Jul 17 03:15:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:33.780Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_daemonset_status_number_unavailable\", asset=\"msh-tsgx11\", asset_id=\"111\", daemonset=\"svclb-traefik-46c02faf\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545659276
Jul 17 03:15:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:33.812Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_labels\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"kubernetes-dashboard\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545659276
Jul 17 03:15:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:33.824Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_running\", asset=\"twa-tsgx05\", asset_id=\"99\", container=\"monit-stream\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"mrzcpd-exporter-5cf7f967cc-plbbr\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"ac696f24-a067-4de4-82cf-63dca2761ac4\"}" timestamp=1689545659894
Jul 17 03:15:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:33.919Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_bytes_total\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth2f4f6888\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545660605
Jul 17 03:15:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:33.933Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_fifo_total\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens6f0np0\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545659093
Jul 17 03:15:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:34.211Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_type\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"k3s-serving\", service=\"prometheus-kube-state-metrics\", type=\"kubernetes.io/tls\"}" timestamp=1689545659894
Jul 17 03:15:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:34.317Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"tmpfs\", name=\"data-var-lib-kubelet-pods-e02c929a\\\\x2d3c07\\\\x2d4477\\\\x2db04a\\\\x2d8ffc16c405f5-volumes-kubernetes.io\\\\x7esecret-kubernetes\\\\x2ddashboard\\\\x2dcerts.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689545667691
Jul 17 03:15:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:34.338Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_packets_dropped_total\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"cadvisor\", interface=\"eth0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689545646080
Jul 17 03:15:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:34.435Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_speed_bytes\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth5dd438f9\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545660605
Jul 17 03:15:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:34.462Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"12\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545660605
Jul 17 03:15:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:34.705Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas_unavailable\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"ipmi-exporter\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545659894
Jul 17 03:15:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:40.739Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", phase=\"Pending\", pod=\"cm-policy-local-cache-redis-master-69967b5879-j7kh6\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"dc48faf7-72ec-4d62-807b-3dca8380f0aa\"}" timestamp=1689545680034
Jul 17 03:15:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:40.758Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545671647
Jul 17 03:15:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:40.765Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"40\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545671647
Jul 17 03:15:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:40.782Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"117\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545671647
Jul 17 03:15:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:40.785Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_vl15_dropped_total\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545671647
Jul 17 03:15:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:40.793Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-e9aa25a33698d95841aab03c5f1d5842f70bead06e8836bd9c0af9d422de0175.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"scope\"}" timestamp=1689545675764
Jul 17 03:15:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:40.889Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_packets_total\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth0ed14f65\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545671647
Jul 17 03:15:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:40.960Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"process_virtual_memory_bytes\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545673827
Jul 17 03:15:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:40.977Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.grpc.v1.cri-sandboxes-6cf38c3f03fc1840db1712a51403cce5aa467bf27b6a1c2efd9bcf9ea5ba4604-shm.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689545675764
Jul 17 03:15:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:41.112Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"5\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545671647
Jul 17 03:15:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:41.112Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_daemonset_metadata_generation\", asset=\"twa-tsgx04\", asset_id=\"97\", daemonset=\"dign-server\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545680034
Jul 17 03:15:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:41.130Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"nsfs\", name=\"run-netns-cni\\\\x2d47aec943\\\\x2dba78\\\\x2d0242\\\\x2d7eeb\\\\x2dc42c2dd31409.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689545675764
Jul 17 03:15:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:41.258Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_link_downed_total\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_11\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545671647
Jul 17 03:15:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:41.270Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2did-wwn\\\\x2d0x62cea7f08346c5002c26f9f8d57b1b40\\\\x2dpart4.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689545675764
Jul 17 03:15:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:41.341Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2dpartlabel-TSG\\\\x2dOS\\\\x2dSYSROOT.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689545675764
Jul 17 03:15:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:41.358Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_annotations\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"coredns-64587b7d8\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545680034
Jul 17 03:15:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:41.360Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_compactionsCompletedCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"assessment_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"c378ae379a0d52c48d9df891757d1b92\", table=\"index_time_assessment_hos_bucket\", type=\"index\"}" timestamp=1689545679756
Jul 17 03:15:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:41.368Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"var-lib-kubelet-pods-2133adaf\\\\x2dcc90\\\\x2d44bb\\\\x2d9cd8\\\\x2de180b9c1e90b-volume\\\\x2dsubpaths-telegraf-telegraf\\\\x2dsecurity-0.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689545675764
Jul 17 03:15:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:41.379Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"getty.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"target\"}" timestamp=1689545675764
Jul 17 03:15:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:41.436Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_maxFlushQueueSize\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"cdead6d6bac6f43a6b32d23108a1fb76\", table=\"proxy_hos_bucket\", type=\"data\"}" timestamp=1689545679756
Jul 17 03:15:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:41.446Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-mrzcpd-exporter-xgz49\", project=\"TSG-OS\", project_id=\"7\", reason=\"Shutdown\", service=\"prometheus-kube-state-metrics\", uid=\"301784b5-1927-45d7-84ae-687804ad7a5e\"}" timestamp=1689545680034
Jul 17 03:15:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:41.465Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_packets_transmitted_total\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_9\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545671647
Jul 17 03:15:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:41.548Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"4\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545671647
Jul 17 03:15:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:41.636Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-a5104a71fcffa4fdb02c774d48755d39d7fbc70c216860da9d967bb75977a92f.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"scope\"}" timestamp=1689545675764
Jul 17 03:15:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:41.756Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_flushesQueuedCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"proxy_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"647386ee248901a50fcc87544d60e617\", table=\"index_time_proxy_hos_bucket\", type=\"index\"}" timestamp=1689545679756
Jul 17 03:15:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:41.809Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"twa-tsgx04\", asset_id=\"97\", container=\"metrics\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"cadvisor\", failure_type=\"pgfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"hierarchy\"}" timestamp=1689545658462
Jul 17 03:15:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:41.825Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_net_dev_group\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens7f1np1\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545671647
Jul 17 03:15:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:41.839Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_PostIndexUpdateTime_min\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"PhoenixIndexer\"}" timestamp=1689545679756
Jul 17 03:15:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:41.896Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_ips\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", ip=\"10.42.0.74\", ip_family=\"4\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-cm-policy-local-cache-rxfmw\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"9e8ca26d-6038-4672-99ec-32891ca961a5\"}" timestamp=1689545680034
Jul 17 03:15:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:41.897Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_incrementCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"8c59e7dcecaf347d8ba67425800f0b96\", table=\"assessment_hos_bucket\", type=\"data\"}" timestamp=1689545679756
Jul 17 03:15:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:41.924Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_filteredReadRequestCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", namespace=\"tsg_galaxy\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"7fed3599680b8cd8d453ece7a92c43c0\", table=\"gtpc_knowledge_base\", type=\"data\"}" timestamp=1689545679756
Jul 17 03:15:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:41.935Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_deleteCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"proxy_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"158238ac589714990cdb5a21858ffe90\", table=\"index_time_proxy_hos_bucket\", type=\"index\"}" timestamp=1689545679756
Jul 17 03:15:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:41.945Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_10_193_1_metric_compactionInputSize_min\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545679756
Jul 17 03:15:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:42.052Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_appendCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"proxy_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"9e185aa8f79610d5667059ec3c98abf3\", table=\"index_time_proxy_hos_bucket\", type=\"index\"}" timestamp=1689545679756
Jul 17 03:15:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:42.201Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"twa-tsgx04\", asset_id=\"97\", condition=\"false\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"systemd-exporter-7f549d9485-dptvd\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"6d31b073-444a-4516-8c0d-9fdeee210f8c\"}" timestamp=1689545680034
Jul 17 03:15:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:42.221Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"twa-tsgx04\", asset_id=\"97\", condition=\"unknown\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"traefik-64f55bb67d-8ff2s\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"7fed6579-2b87-448c-8a58-d522c3843522\"}" timestamp=1689545680034
Jul 17 03:15:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:42.241Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"22\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545671647
Jul 17 03:15:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:42.550Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", cpu=\"93\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545676242
Jul 17 03:15:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:42.588Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_files_free\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"tmpfs\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545676242
Jul 17 03:15:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:42.797Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"initrd.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"target\"}" timestamp=1689545680671
Jul 17 03:15:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:42.812Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"lvm2-lvmpolld.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"service\"}" timestamp=1689545680671
Jul 17 03:15:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:42.822Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_up\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens7f1np1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545676242
Jul 17 03:15:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:42.852Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", container=\"prometheus-server-configmap-reload\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"sleeping\"}" timestamp=1689545670115
Jul 17 03:15:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:42.866Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_created\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.cm-policy-local-cache.v1\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545675154
Jul 17 03:15:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:42.913Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_status_start_time\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", job_name=\"helm-install-configmap-reloader\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545675154
Jul 17 03:15:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:43.007Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_carrier_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens2f0np0\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545676242
Jul 17 03:15:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:43.175Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"kube-multus-ds-t9bhj\", project=\"TSG-OS\", project_id=\"7\", reason=\"Evicted\", service=\"prometheus-kube-state-metrics\", uid=\"095d5887-0ff1-468e-9463-a3765c458177\"}" timestamp=1689545675154
Jul 17 03:15:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:43.243Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_forks_total\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Gohangout-msh-dellr740-dt002\", endpoint_id=\"167\", module=\"DC-Gohangout\", module_id=\"35\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545682624
Jul 17 03:15:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:43.262Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-1d9c1047\\\\x2d30bc\\\\x2d4de8\\\\x2dbd00\\\\x2d07e7917cd6b7-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2dng2nx.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689545680671
Jul 17 03:15:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:43.286Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_id\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"em1\", endpoint=\"NC-Gohangout-msh-dellr740-dt002\", endpoint_id=\"167\", module=\"DC-Gohangout\", module_id=\"35\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545682624
Jul 17 03:15:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:43.313Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_virtual_memory_max_bytes\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"mrzcpd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689545680671
Jul 17 03:15:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:43.356Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689545680671
Jul 17 03:15:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:43.392Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_scrape_collector_duration_seconds\", asset=\"pcap-tsgx01\", asset_id=\"71\", collector=\"zfs\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545676242
Jul 17 03:15:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:43.624Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-a38bc0a0\\\\x2df54d\\\\x2d4548\\\\x2db0f1\\\\x2d6472f0282ca8-volume\\\\x2dsubpaths-config\\\\x2dvolume-telegraf\\\\x2dsecurity-2.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689545680671
Jul 17 03:15:44 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:44.350Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", cpu=\"33\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Gohangout-msh-dellr740-dt002\", endpoint_id=\"167\", module=\"DC-Gohangout\", module_id=\"35\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545682624
Jul 17 03:15:45 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:45.966Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"java_lang_OperatingSystem_ProcessCpuTime\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-Yarn-pcap-dellr740-dt001-ResourceManager\", endpoint_id=\"135\", module=\"DC-Yarn\", module_id=\"1\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545685217
Jul 17 03:15:48 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:48.546Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_ScanTime_99th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Server\"}" timestamp=1689545687438
Jul 17 03:15:48 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:48.560Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_putCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"06563a29a592feca015768fc830f32a6\", table=\"assessment_hos_bucket\", type=\"data\"}" timestamp=1689545687438
Jul 17 03:15:48 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:48.576Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_incrementCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"firewall_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"efdc1c66d4c62e1cb4d0bf16a4f8d36c\", table=\"index_partfile_firewall_hos_bucket\", type=\"index\"}" timestamp=1689545687438
Jul 17 03:15:48 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:48.594Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_194_3_metric_putTime_90th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545687438
Jul 17 03:15:48 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:48.853Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_2_metric_compactionOutputFileCount_75th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545687438
Jul 17 03:15:48 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:48.927Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_scanCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"c525897a589ef585c72bc9dd40a14405\", table=\"firewall_hos_bucket\", type=\"data\"}" timestamp=1689545687438
Jul 17 03:15:48 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:48.942Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_193_1_metric_compactionOutputFileCount_95th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545687438
Jul 17 03:15:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:49.021Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_193_1_metric_majorCompactionInputFileCount_75th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545687438
Jul 17 03:15:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:49.023Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_194_1_metric_splitTime_min\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545687438
Jul 17 03:15:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:49.033Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_hbase_table_rsgroup_metric_putTime_max\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545687438
Jul 17 03:15:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:49.037Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_getCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"assessment_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"840c37e0ce972c3ea886c0cf013bc466\", table=\"index_filename_assessment_hos_bucket\", type=\"index\"}" timestamp=1689545687438
Jul 17 03:15:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:49.117Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_4_metric_flushOutputSize_max\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545687438
Jul 17 03:15:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:49.119Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_putCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"session_record_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"3e1c7aa641482ceb8e28b5526f23c5e9\", table=\"index_filename_session_record_hos_bucket\", type=\"index\"}" timestamp=1689545687438
Jul 17 03:15:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:49.139Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_scanCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"assessment_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"5d0defd99486c92aa30397e891e99c71\", table=\"index_partfile_assessment_hos_bucket\", type=\"index\"}" timestamp=1689545687438
Jul 17 03:15:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:49.154Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_maxFlushQueueSize\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"firewall_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"efdc1c66d4c62e1cb4d0bf16a4f8d36c\", table=\"index_partfile_firewall_hos_bucket\", type=\"index\"}" timestamp=1689545687438
Jul 17 03:15:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:49.170Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_compactionsFailedCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"rtp_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"8b4bb800fda3b83e73863db9941293cf\", table=\"index_partfile_rtp_hos_bucket\", type=\"index\"}" timestamp=1689545687438
Jul 17 03:15:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:49.447Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_appendCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", namespace=\"SYSTEM\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"d330648f7279c8a62731bd0e25e320c5\", table=\"LOG\", type=\"data\"}" timestamp=1689545687438
Jul 17 03:15:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:49.465Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_194_3_metric_splitTime_99th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545687438
Jul 17 03:15:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:49.488Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_getCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"assessment_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"a7a5c13a400401472e2d36778e8ca3a1\", table=\"index_time_assessment_hos_bucket\", type=\"index\"}" timestamp=1689545687438
Jul 17 03:15:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:49.579Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_UnblockedFlushes_25th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Memory\"}" timestamp=1689545687438
Jul 17 03:15:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:49.584Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_4_metric_maxStoreFileAge\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545687438
Jul 17 03:15:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:49.611Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_ResponseSize_98th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"IPC\"}" timestamp=1689545687438
Jul 17 03:15:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:49.892Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-pushgateway-bd476bcf9-96z55\", project=\"TSG-OS\", project_id=\"7\", reason=\"NodeLost\", service=\"prometheus-kube-state-metrics\", uid=\"1617bf89-68ca-42e6-8a6e-a79f76bf18ce\"}" timestamp=1689545681568
Jul 17 03:15:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:49.897Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_slowlog_length\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545686625
Jul 17 03:15:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:49.909Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_owner\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", owner_is_controller=\"true\", owner_kind=\"Job\", owner_name=\"helm-install-mrzcpd-exporter\", pod=\"helm-install-mrzcpd-exporter-zjhth\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"a88ca435-a295-43d3-bffd-a3842ce86c5e\"}" timestamp=1689545681568
Jul 17 03:15:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:49.932Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2dpartuuid-d70216a5\\\\x2db5f8\\\\x2d4564\\\\x2d93f9\\\\x2dec11144d824d.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689545681057
Jul 17 03:15:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:49.945Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_metadata_generation\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"local-path-provisioner-957fdf8bc\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545681568
Jul 17 03:15:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:49.980Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_colls_total\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens3f1np1\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545681797
Jul 17 03:15:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:50.004Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:c0-0000:c0:01.1-0000:c1:00.0-infiniband_mad-umad8.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689545681057
Jul 17 03:15:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:50.016Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"123\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545686709
Jul 17 03:15:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:50.022Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-virtual-net-bond0.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689545681057
Jul 17 03:15:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:50.032Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"4\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545686709
Jul 17 03:15:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:50.110Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_file_descriptors\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"kube-state-metrics\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545670704
Jul 17 03:15:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:50.146Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_oom_events_total\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"exporter-merger\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545665046
Jul 17 03:15:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:50.169Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_status_succeeded\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", job_name=\"helm-install-configmap-reloader\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545681568
Jul 17 03:15:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:50.190Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_compactionsQueuedCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"session_record_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"88a9bbe2136852f7ad790e6c839f28d2\", table=\"index_time_session_record_hos_bucket\", type=\"index\"}" timestamp=1689545687438
Jul 17 03:15:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:50.303Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_limit_bytes\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/1d0a9283baaeb68215db2c28cfe83a4ec2014931320cc1ec38d111b6cd89d7b3/shm\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545687342
Jul 17 03:15:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:50.320Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-pod7be94d73_b8c6_4b83_adb9_10dac35bf41e.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"slice\"}" timestamp=1689545681057
Jul 17 03:15:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:50.451Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"twa-tsgx08\", asset_id=\"105\", container=\"ipmi-exporter\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"iowaiting\"}" timestamp=1689545682298
Jul 17 03:15:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:50.535Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_metadata_generation\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"tsg-traffic-engine-numa-1-firewall\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545681568
Jul 17 03:15:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:50.654Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_ready\", asset=\"twa-tsgx08\", asset_id=\"105\", container=\"exporter-merger\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"mrzcpd-exporter-5cf7f967cc-b62w2\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"c4ec4f52-224c-4f5b-9432-db80b6600279\"}" timestamp=1689545681568
Jul 17 03:15:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:50.670Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-ens8f1np1.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689545686554
Jul 17 03:15:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:50.673Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"13\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545686709
Jul 17 03:15:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:50.778Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"16\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545686709
Jul 17 03:15:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:50.879Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_up\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens7f0np0\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545686709
Jul 17 03:15:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:50.919Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_file_descriptors\", asset=\"twa-tsgx08\", asset_id=\"105\", container=\"prometheus-pushgateway\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545676288
Jul 17 03:15:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:51.035Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_annotations\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-master\", uid=\"a8b7c5af-aad2-4b80-8942-f54cbe43a552\"}" timestamp=1689545679668
Jul 17 03:15:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:51.050Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx10\", asset_id=\"109\", cpu=\"82\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545681797
Jul 17 03:15:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:51.177Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"twa-tsgx08\", asset_id=\"105\", condition=\"true\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-systemd-exporter-ns8p4\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"132271d2-be10-4fb8-8ec9-3f73cc203cb2\"}" timestamp=1689545681568
Jul 17 03:15:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:51.194Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_frequency_max_hertz\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"49\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545686709
Jul 17 03:15:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:51.210Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"eno34np1\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545686709
Jul 17 03:15:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:51.261Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_packets_dropped_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", interface=\"ens8f0np0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545687342
Jul 17 03:15:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:51.272Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_ready\", asset=\"twa-tsgx08\", asset_id=\"105\", container=\"local-path-provisioner\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"local-path-provisioner-957fdf8bc-486df\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"59e0fcfb-c5b9-4bb0-aefc-a15fed75746e\"}" timestamp=1689545681568
Jul 17 03:15:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:51.350Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_info\", asset=\"twa-tsgx10\", asset_id=\"109\", created_by_kind=\"ReplicaSet\", created_by_name=\"metrics-server-54dc485875\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", host_ip=\"10.10.30.170\", host_network=\"false\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", node=\"localhost\", nz_agent_id=\"3\", pod=\"metrics-server-54dc485875-8pvbg\", pod_ip=\"10.42.0.107\", priority_class=\"system-node-critical\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"c328283a-7db9-4ea0-a4d5-94b0b13c63dd\"}" timestamp=1689545679668
Jul 17 03:15:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:51.373Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", cpu=\"29\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-olap003\", endpoint_id=\"235\", mode=\"steal\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"yarn_resourcemanager,zookeeper,nacos,kafka,spark,druid_data,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545689713
Jul 17 03:15:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:51.607Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_info\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"chart-values-tsg-traffic-engine-numa-0\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545681568
Jul 17 03:15:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:52.515Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_size_bytes\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"shm\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/d723a6b63b9c0f46829212b5ec76cef76cfd9f138a81d796dfeb1fc977ecfb33/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545689337
Jul 17 03:15:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:52.533Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_info\", address=\"08:c0:eb:59:d3:24\", asset=\"pcap-tsgx11\", asset_id=\"115\", broadcast=\"ff:ff:ff:ff:ff:ff\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens7f0np0\", duplex=\"full\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operstate=\"up\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545689337
Jul 17 03:15:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:52.601Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_packets_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens6f0np0\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545689337
Jul 17 03:15:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:52.634Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", cpu=\"30\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545689337
Jul 17 03:15:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:52.791Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled_time\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"kube-multus-ds-rvzdl\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"ac103daf-2fd5-4026-ae82-9b9eca69f2cd\"}" timestamp=1689545684478
Jul 17 03:15:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:52.915Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_name_assign_type\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"vethf7db82f7\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545689337
Jul 17 03:15:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:52.947Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-9b559a6c\\\\x2d7e7e\\\\x2d492f\\\\x2daeb3\\\\x2df47a8be58f12-volumes-kubernetes.io\\\\x7esecret-cm\\\\x2dcache\\\\x2dextra\\\\x2dsecrets.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689545685496
Jul 17 03:15:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:53.223Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_info\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"chart-values-mrzcpd-exporter\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545684478
Jul 17 03:15:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:53.296Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_data_transmitted_bytes_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"mlx5_5\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545689337
Jul 17 03:15:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:53.479Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_complete\", asset=\"msh-tsgx12\", asset_id=\"113\", condition=\"false\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", job_name=\"helm-install-tsg-traffic-engine-numa-1\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545688663
Jul 17 03:15:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:53.490Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db137\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545679367
Jul 17 03:15:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:53.519Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", phase=\"Pending\", pod=\"dign-server-ml4qh\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"2ed77d82-be58-47ef-9e9a-55c3cc7c3703\"}" timestamp=1689545688663
Jul 17 03:15:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:53.642Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"msh-tsgx08\", asset_id=\"65\", cpu=\"13\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545688328
Jul 17 03:15:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:53.651Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cpu=\"45\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545679409
Jul 17 03:15:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:53.674Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx08\", asset_id=\"65\", container=\"local-path-provisioner\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Sync\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545664800
Jul 17 03:15:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:53.739Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", container=\"dign-server-dns\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Discard\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545690109
Jul 17 03:15:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:53.750Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"msh-tsgx08\", asset_id=\"65\", condition=\"true\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"coredns-749bdddcdf-lzwfk\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"a715b3b6-710d-4f95-8f01-a1d211f1bef3\"}" timestamp=1689545688950
Jul 17 03:15:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:53.757Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cpu=\"21\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545679409
Jul 17 03:15:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:53.813Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-sysusers.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689545685496
Jul 17 03:15:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:53.851Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_cpu_period\", asset=\"pcap-tsgx07\", asset_id=\"85\", container=\"monit-stream\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545690608
Jul 17 03:15:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:53.872Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_carrier_total\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"eno2\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545688328
Jul 17 03:15:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:54.049Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"72\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545679409
Jul 17 03:15:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:54.053Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", cpu=\"102\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545689337
Jul 17 03:15:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:54.066Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_tasks_current\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-udev-settle.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689545692135
Jul 17 03:15:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:54.101Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_hwmon_sensor_label\", asset=\"pcap-tsgx07\", asset_id=\"85\", chip=\"pci0000:00_0000:00:18_3\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", label=\"tctl\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", sensor=\"temp1\", service=\"prometheus-node-exporter\"}" timestamp=1689545679409
Jul 17 03:15:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:54.144Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", cpu=\"56\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545689337
Jul 17 03:15:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:54.144Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_resident_memory_bytes\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"rdma-ndd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689545680884
Jul 17 03:15:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:54.163Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_size_bytes\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"tmpfs\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545688328
Jul 17 03:15:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:54.233Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Tcp_Bypass_STM\", app_name=\"sapp4\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", vsysId=\"1\"}" timestamp=1689545687223
Jul 17 03:15:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:54.252Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-c6171e2c08ab45cf801de20576b6cb819bb0cb1ed492b3668dad3b407d175a73.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"scope\"}" timestamp=1689545685496
Jul 17 03:15:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:54.257Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"pcap-tsgx07\", asset_id=\"85\", condition=\"unknown\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", pod=\"kubernetes-dashboard-66dd5b456-rmvl8\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"d360d8a3-9aa3-40d2-ad94-ccc470a435ce\"}" timestamp=1689545682058
Jul 17 03:15:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:54.267Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-delete-tsg-traffic-engine-numa-1-8tjrj\", project=\"TSG-OS\", project_id=\"7\", reason=\"Evicted\", service=\"prometheus-kube-state-metrics\", uid=\"9061244f-a573-4738-a662-2e823d1f8614\"}" timestamp=1689545688663
Jul 17 03:15:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:54.285Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_drop_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens4f2np2\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545679409
Jul 17 03:15:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:54.301Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx12\", asset_id=\"113\", cpu=\"50\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545683391
Jul 17 03:15:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:54.311Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_edac_uncorrectable_errors_total\", asset=\"msh-tsgx12\", asset_id=\"113\", controller=\"1\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545683391
Jul 17 03:15:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:54.326Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-tmpfiles-setup.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689545691179
Jul 17 03:15:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:54.464Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx12\", asset_id=\"113\", cpu=\"42\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545683391
Jul 17 03:15:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:54.465Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cpu=\"93\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545679409
Jul 17 03:15:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:54.492Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db249\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545685184
Jul 17 03:15:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:54.520Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-e161ee9508fcf6ccace29a38430157e4a6455775d22692fedb1ee5baa9b57345-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689545692135
Jul 17 03:15:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:54.626Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"mlocate-updatedb.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"simple\", type=\"service\"}" timestamp=1689545680884
Jul 17 03:15:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:54.694Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_labels\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"ipmi-exporter\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545682058
Jul 17 03:15:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:54.700Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-2d58616a1e2ba5df115c2438e56048be6927c43a5371eb4cc637d41e744d4d5b-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689545692135
Jul 17 03:15:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:54.711Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Tcp_Bit\", app_name=\"sapp4\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", vsysId=\"1\"}" timestamp=1689545684783
Jul 17 03:15:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:54.718Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"selinux-autorelabel-mark.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"service\"}" timestamp=1689545692135
Jul 17 03:15:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:54.735Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx12\", asset_id=\"113\", cpu=\"19\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545683391
Jul 17 03:15:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:54.753Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cpu=\"34\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545679409
Jul 17 03:15:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:54.777Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"emergency.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"service\"}" timestamp=1689545680884
Jul 17 03:15:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:54.781Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_size_bytes\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"shm\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/25bdf55ad466762b6795a0631abb4c86452da514a9f3b40469e16bef2abcf59f/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545688328
Jul 17 03:15:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:54.800Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_device_id\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"bond0\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545688328
Jul 17 03:15:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:54.903Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-0fcb989adc91a69fee12c95648cde21b1da3a348b3b80ff735fbe6e5411b8c50-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689545680884
Jul 17 03:15:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:54.908Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_labels\", asset=\"msh-tsgx12\", asset_id=\"113\", configmap=\"chart-content-tsg-traffic-engine-numa-0\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545688663
Jul 17 03:15:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:54.927Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_ready\", asset=\"msh-tsgx12\", asset_id=\"113\", container=\"dign-client\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"dign-client-w2dmk\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"a70ba7d6-c83e-453e-962c-f10b0073510f\"}" timestamp=1689545688663
Jul 17 03:15:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:54.929Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"msh-tsgx08\", asset_id=\"65\", cpu=\"22\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545688328
Jul 17 03:15:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:54.996Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx12\", asset_id=\"113\", cpu=\"98\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545683391
Jul 17 03:15:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:55.010Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_name_assign_type\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"br_dign_s\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545683391
Jul 17 03:15:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:55.047Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_swap\", asset=\"msh-tsgx12\", asset_id=\"113\", container=\"prometheus-server\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545677862
Jul 17 03:15:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:55.087Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_scrape_collector_duration_seconds\", asset=\"msh-tsgx12\", asset_id=\"113\", collector=\"filesystem\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545683391
Jul 17 03:15:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:55.102Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-pod65e70859_0f96_46f5_bc47_05f2ff088f29.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"slice\"}" timestamp=1689545691179
Jul 17 03:15:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:56.412Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_spec_replicas\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-0-firewall-c88ff644d\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545693389
Jul 17 03:15:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:57.114Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_complete\", asset=\"pcap-tsgx03\", asset_id=\"73\", condition=\"true\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", job_name=\"helm-install-prometheus\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545693389
Jul 17 03:15:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:57.131Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"91\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545685883
Jul 17 03:15:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:57.388Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-8a7b184f5c770578def5a10ac44762bd090b8904b928ec7a864895fc40e5d46d.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"scope\"}" timestamp=1689545683372
Jul 17 03:15:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:57.862Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_fifo_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"vethdfe7d9ac\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545685883
Jul 17 03:15:58 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:58.002Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_threads\", asset=\"pcap-tsgx03\", asset_id=\"73\", container=\"kube-multus\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545683278
Jul 17 03:15:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:15:59.971Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-rdma-devices-mlx5_9:1-umad.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689545684879
Jul 17 03:16:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:00.170Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"var-lib-kubelet-pods-b3efc35a\\\\x2d4844\\\\x2d47d9\\\\x2d82c9\\\\x2d7d335d0d4eaa-volume\\\\x2dsubpaths-config\\\\x2dvolume-telegraf-1.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689545684879
Jul 17 03:16:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:00.356Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-4dbd7eb51e70749a083dad8d4909e543ee021e77044334497de7f2402622711c.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"scope\"}" timestamp=1689545684879
Jul 17 03:16:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:01.336Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_info\", asset=\"msh-tsgx01\", asset_id=\"51\", board_id=\"MT_0000000359\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_4\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", firmware_version=\"22.28.4000\", hca_type=\"MT4125\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545689083
Jul 17 03:16:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:01.457Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_owner\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", owner_is_controller=\"true\", owner_kind=\"Job\", owner_name=\"helm-install-tsg-traffic-engine-numa-0\", pod=\"helm-install-tsg-traffic-engine-numa-0-j6fjb\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"0744435a-4d10-4030-a4e0-c03d8d48b916\"}" timestamp=1689545687425
Jul 17 03:16:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:03.358Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_swap\", asset=\"msh-tsgx02\", asset_id=\"63\", container=\"exporter\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545675623
Jul 17 03:16:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:03.827Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"msh-tsgx02\", asset_id=\"63\", container=\"firewall\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", state=\"stopped\"}" timestamp=1689545677388
Jul 17 03:16:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:03.835Z caller=dedupe.go:112 component=remote level=info remote_name=52db6f url=http://10.10.10.159:10090/nz-agent/prometheus/proxy/api/v1/write msg="Dropped sample for series that was not explicitly dropped via relabelling" ref=4436650
Jul 17 03:16:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:03.849Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2dlabel-ONIE\\\\x2dBOOT.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689545688447
Jul 17 03:16:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:04.083Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"phy_tx_packets_total\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", dev=\"virtio_dign_s\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"mrzcpd-exporter\"}" timestamp=1689545697995
Jul 17 03:16:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:04.223Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_threads_max\", asset=\"msh-tsgx02\", asset_id=\"63\", container=\"lb-tcp-80\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545680376
Jul 17 03:16:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:04.229Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", cpu=\"4\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-dt001\", endpoint_id=\"225\", mode=\"nice\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545702988
Jul 17 03:16:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:04.239Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_rss\", asset=\"msh-tsgx02\", asset_id=\"63\", container=\"monit-stream\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545681899
Jul 17 03:16:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:04.243Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-remount-fs.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689545688447
Jul 17 03:16:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:04.247Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx02\", asset_id=\"63\", cpu=\"92\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545698318
Jul 17 03:16:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:04.262Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"prometheus_target_sync_failed_total\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", instance=\"10.10.20.159:10090\", scrape_job=\"103\", type=\"global\"}" timestamp=1689545702990
Jul 17 03:16:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:04.278Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"mlocate-updatedb.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"service\"}" timestamp=1689545688447
Jul 17 03:16:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:04.295Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-virtual-net-veth61606a98.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689545688447
Jul 17 03:16:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:04.303Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx02\", asset_id=\"63\", container=\"telegraf-security\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Sync\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689545688784
Jul 17 03:16:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:04.320Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"msh-tsgx02\", asset_id=\"63\", container=\"telegraf-security\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", state=\"running\"}" timestamp=1689545688784
Jul 17 03:16:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:04.396Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_avail_bytes\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"shm\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/b892f19f35026896fa8a386b989f05457b0f27963f2c1041d563cc0b1c9b984a/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545698318
Jul 17 03:16:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:04.444Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-1dda3932\\\\x2d520e\\\\x2d4976\\\\x2d8890\\\\x2da1a3ca5c9e50-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2d447wb.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689545688447
Jul 17 03:16:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:04.486Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"rescue.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"service\"}" timestamp=1689545688447
Jul 17 03:16:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:04.693Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"net_conntrack_dialer_conn_established_total\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", dialer_name=\"asset_ping_35\", instance=\"10.10.20.159:10090\", type=\"global\"}" timestamp=1689545702990
Jul 17 03:16:05 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:05.064Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-dellr740-olap005\", asset_id=\"19\", cpu=\"29\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Spark-msh-dellr740-olap005\", endpoint_id=\"17\", mode=\"nice\", module=\"NC-Spark\", module_id=\"47\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545703391
Jul 17 03:16:05 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:05.076Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_memory_Hugepagesize_bytes\", asset=\"msh-dellr740-olap005\", asset_id=\"19\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Spark-msh-dellr740-olap005\", endpoint_id=\"17\", module=\"NC-Spark\", module_id=\"47\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545703391
Jul 17 03:16:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:06.628Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-daf8a71de8eae576b828e6e98135123fd676eb5bfb4f49e800becccecf89847e.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"scope\"}" timestamp=1689545693582
Jul 17 03:16:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:06.696Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_drop_total\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth35124140\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545703910
Jul 17 03:16:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:06.739Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cpu=\"3\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545703910
Jul 17 03:16:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:06.767Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cpu=\"98\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545703910
Jul 17 03:16:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:06.788Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2dpartuuid-b092ee06\\\\x2dfd14\\\\x2d4cc9\\\\x2db757\\\\x2de6f8ef569e3f.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689545693582
Jul 17 03:16:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:06.802Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_sockstat_UDP_mem\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545703910
Jul 17 03:16:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:06.817Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Udp_Link_New\", app_name=\"sapp4\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", vsysId=\"1\"}" timestamp=1689545697817
Jul 17 03:16:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:07.039Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cpu=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545703910
Jul 17 03:16:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:07.052Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_frame_total\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens3f0np0\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545703910
Jul 17 03:16:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:07.200Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_created\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"dashboard-metrics-scraper-5b94897dc9\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545701227
Jul 17 03:16:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:07.241Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_hwmon_sensor_label\", asset=\"twa-tsgx01\", asset_id=\"91\", chip=\"pci0000:00_0000:00:19_3\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", label=\"tctl\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", sensor=\"temp1\", service=\"prometheus-node-exporter\"}" timestamp=1689545703910
Jul 17 03:16:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:07.629Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_packets_total\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth444895a4\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545703910
Jul 17 03:16:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:07.775Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_running\", asset=\"twa-tsgx01\", asset_id=\"91\", container=\"monit-stream\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"mrzcpd-exporter-5cf7f967cc-97flx\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"083e9092-7400-4cad-afc0-cffbf05464a4\"}" timestamp=1689545701227
Jul 17 03:16:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:08.172Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cpu=\"78\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545703910
Jul 17 03:16:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:08.245Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-ens8f1np1.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689545693582
Jul 17 03:16:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:08.279Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_bytes_total\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth1c375d86\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545703910
Jul 17 03:16:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:08.299Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_drop_total\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"eno1\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545703910
Jul 17 03:16:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:10.134Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_storeFileSize\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"firewall_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"c2db334522d1026cb494aa8837853784\", table=\"index_time_firewall_hos_bucket\", type=\"index\"}" timestamp=1689545708804
Jul 17 03:16:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:10.166Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_avgStoreFileAge\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"default\", project=\"OLAP\", project_id=\"3\", region=\"b80f77b3ae41c2c5dc46743c217a463a\", table=\"index_filename_default\", type=\"index\"}" timestamp=1689545708804
Jul 17 03:16:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:10.189Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_appendCount\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"proxy_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"e384599f99b40404328512097fa0536f\", table=\"index_filename_proxy_hos_bucket\", type=\"index\"}" timestamp=1689545708804
Jul 17 03:16:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:10.205Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_numActivePriorityHandler\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"IPC\"}" timestamp=1689545708804
Jul 17 03:16:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:11.474Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_getCount\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", module=\"DC-HBase\", module_id=\"3\", namespace=\"SYSTEM\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"0e351c81d54c0e5eb9bf16f6dab58202\", table=\"LOG\", type=\"data\"}" timestamp=1689545708804
Jul 17 03:16:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:11.674Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_194_2_metric_compactionOutputSize_98th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545710149
Jul 17 03:16:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:11.683Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_storeFileSize\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"knowledge_base_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"c95a264dbee52b8d15a6e132ecfd0800\", table=\"index_time_knowledge_base_hos_bucket\", type=\"index\"}" timestamp=1689545710149
Jul 17 03:16:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:11.699Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_lastMajorCompactionAge\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"rtp_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"9aaf17714a26df7ee89beb812a9e96b3\", table=\"index_filename_rtp_hos_bucket\", type=\"index\"}" timestamp=1689545710149
Jul 17 03:16:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:11.709Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_1_metric_compactionOutputSize_25th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545710149
Jul 17 03:16:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:11.715Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_194_4_metric_majorCompactionOutputSize_25th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545710149
Jul 17 03:16:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:11.785Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_readRequestCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"assessment_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"db3245bccdc6403a0161f317bbd97ed6\", table=\"index_filename_assessment_hos_bucket\", type=\"index\"}" timestamp=1689545710149
Jul 17 03:16:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:11.919Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_default_metric_flushTime_99th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545710149
Jul 17 03:16:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:12.064Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_gc_duration_seconds_count\", asset=\"twa-dellr740-dt002\", asset_id=\"39\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"NC-Gohangout-twa-dellr740-dt002\", endpoint_id=\"179\", module=\"DC-Gohangout\", module_id=\"35\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545711525
Jul 17 03:16:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:12.168Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_194_3_metric_incrementTime_90th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545710149
Jul 17 03:16:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:12.324Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_device_id\", asset=\"twa-dellr740-dt002\", asset_id=\"39\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"em4\", endpoint=\"OLAP-Node-Exporter-twa-dellr740-dt002\", endpoint_id=\"259\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545712056
Jul 17 03:16:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:12.864Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_RequestSize_SizeRangeCount_100_1000\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"IPC\"}" timestamp=1689545710149
Jul 17 03:16:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:12.935Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_mallocs_total\", asset=\"twa-dellr740-dt002\", asset_id=\"39\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"OLAP-Node-Exporter-twa-dellr740-dt002\", endpoint_id=\"259\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545712056
Jul 17 03:16:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:12.935Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_1_metric_majorCompactionOutputFileCount_75th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545710149
Jul 17 03:16:13 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:13.244Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_194_1_metric_tableWriteQueryPerSecond_15min_rate\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545710149
Jul 17 03:16:13 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:13.269Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_1_metric_avgStoreFileAge\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545710149
Jul 17 03:16:13 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:13.383Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_TASK_metric_incrementTime_99_9th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545710149
Jul 17 03:16:17 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:17.246Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"111\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545701230
Jul 17 03:16:18 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:18.703Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"clickhouse_arena_alloc_bytes_total\", asset=\"twa-dellr740-dt003\", asset_id=\"41\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"NC-Clickhouse-twa-dellr740-dt003\", endpoint_id=\"91\", module=\"NC-Clickhouse\", module_id=\"29\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545718300
Jul 17 03:16:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:19.369Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"clickhouse_block_discard_ops_sdb\", asset=\"twa-dellr740-dt003\", asset_id=\"41\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"NC-Clickhouse-twa-dellr740-dt003\", endpoint_id=\"91\", module=\"NC-Clickhouse\", module_id=\"29\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545718300
Jul 17 03:16:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:19.939Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_data_received_bytes_total\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_3\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545717612
Jul 17 03:16:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:20.021Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx06\", asset_id=\"101\", container=\"dashboard-metrics-scraper\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Read\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545712549
Jul 17 03:16:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:20.052Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"clevis-luks-askpass.path\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"path\"}" timestamp=1689545704356
Jul 17 03:16:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:20.056Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_labels\", asset=\"twa-tsgx06\", asset_id=\"101\", configmap=\"certstore-tsg-traffic-engine-numa-0\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545713949
Jul 17 03:16:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:20.192Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"108\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545717612
Jul 17 03:16:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:20.200Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"118\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545717612
Jul 17 03:16:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:20.211Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"52\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545717612
Jul 17 03:16:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:20.276Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_bytes_total\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens1f1np1\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545717612
Jul 17 03:16:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:20.298Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_mtu_bytes\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth7eb6ef28\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545717612
Jul 17 03:16:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:20.304Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"cadvisor_version_info\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"cadvisor\", kernelVersion=\"5.17.15-1.el8.x86_64\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", osVersion=\"TSG-OS v23.07-rc1-079e964 (TSGXNXR620G40R01P0906)\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545715735
Jul 17 03:16:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:20.321Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_bytes_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"vethd6f33409\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545708263
Jul 17 03:16:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:20.363Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"26\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545708263
Jul 17 03:16:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:20.368Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"79\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545717612
Jul 17 03:16:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:20.390Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_compressed_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"vethd6f33409\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545708263
Jul 17 03:16:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:20.390Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda5\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", fstype=\"ext4\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/var/lib/kubelet/pods/ab53e5a7-f22e-498d-b5af-535ae7359350/volume-subpaths/telegraf/telegraf-security/0\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545708263
Jul 17 03:16:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:20.392Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"83\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545708263
Jul 17 03:16:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:20.406Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-ens4f2np2.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689545710657
Jul 17 03:16:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:20.410Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db107\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545711998
Jul 17 03:16:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:20.435Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_max_fds\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"hasp_monitor.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689545704356
Jul 17 03:16:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:20.441Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"rescue.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689545710657
Jul 17 03:16:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:20.451Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_created\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"tsg-traffic-engine-numa-0-firewall\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545713828
Jul 17 03:16:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:20.469Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"84\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545708263
Jul 17 03:16:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:20.552Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", container=\"coredns\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Write\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545711534
Jul 17 03:16:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:20.563Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_metadata_resource_version\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.tsg-traffic-engine-numa-0.v4\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545713828
Jul 17 03:16:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:20.581Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_memory_WritebackTmp_bytes\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545708263
Jul 17 03:16:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:20.595Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_protocol_type\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth8d94c437\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545708263
Jul 17 03:16:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:20.597Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dracut-pre-pivot.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689545710657
Jul 17 03:16:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:20.610Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-3a00f506971912e91a0f2421a4cd5f23450e9b733c343cbedf66427667057151.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"scope\"}" timestamp=1689545704356
Jul 17 03:16:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:20.633Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-95bc4fcc9e5e23097a950e3b31a5e3c8a263a6882a0f9b5ed6de93811d371f00.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"scope\"}" timestamp=1689545710657
Jul 17 03:16:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:20.653Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"11\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545708263
Jul 17 03:16:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:20.672Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_owner\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", owner_is_controller=\"true\", owner_kind=\"Deployment\", owner_name=\"prometheus-pushgateway\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"prometheus-pushgateway-bd476bcf9\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545713828
Jul 17 03:16:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:20.675Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2dpath-pci\\\\x2d0000:81:00.0\\\\x2dscsi\\\\x2d0:2:0:0\\\\x2dpart3.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689545704356
Jul 17 03:16:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:20.693Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"67\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545708263
Jul 17 03:16:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:20.702Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_scrape_collector_duration_seconds\", asset=\"pcap-tsgx10\", asset_id=\"89\", collector=\"netdev\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545708263
Jul 17 03:16:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:20.730Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-8f7f8c0558593a7bf6e79598104df942fc87a8f0bc9a7d57b6f3d47eadcf9d3e.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"scope\"}" timestamp=1689545710657
Jul 17 03:16:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:20.750Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db183\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545714840
Jul 17 03:16:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:20.770Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_file_descriptors\", asset=\"twa-tsgx06\", asset_id=\"101\", container=\"dashboard-metrics-scraper\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545712549
Jul 17 03:16:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:20.792Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_alloc_bytes_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545708263
Jul 17 03:16:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:20.805Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-91db220ff3ff66fa5f045151cce1c92c7ef5aac0a317a5e1affaf35caa0fee5d.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"scope\"}" timestamp=1689545710657
Jul 17 03:16:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:20.815Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545717612
Jul 17 03:16:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:20.844Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2did-wwn\\\\x2d0x62cea7f0add598002c2c8d3154fff241\\\\x2dpart3.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689545710657
Jul 17 03:16:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:20.860Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kmod-static-nodes.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"service\"}" timestamp=1689545710657
Jul 17 03:16:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:20.895Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", container=\"dign-server-dns\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"stopped\"}" timestamp=1689545704592
Jul 17 03:16:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:20.909Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"shm\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/0cbba4d97a73cf1cf108a44b70aecad491f4b19a4049279ba4e8f9238ce32d97/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545708263
Jul 17 03:16:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:20.921Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_free_bytes\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"shm\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/aebe77d3b5b78e4f598e7b51fd01ec5ef6e9c50bb0cb2dc1010f3ecfe3e703a8/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545708263
Jul 17 03:16:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:20.943Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", container=\"coredns\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Sync\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545711534
Jul 17 03:16:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:20.955Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_user_seconds_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", container=\"monit-stream\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545702938
Jul 17 03:16:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:21.082Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_packets_total\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethfc45064b\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545717612
Jul 17 03:16:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:21.091Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"rsyslog.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689545704356
Jul 17 03:16:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:21.093Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"123\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545708263
Jul 17 03:16:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:21.099Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_tasks_max\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"iscsid.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689545704356
Jul 17 03:16:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:21.112Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_node_status_capacity\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", node=\"localhost\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", resource=\"cpu\", service=\"prometheus-kube-state-metrics\", unit=\"core\"}" timestamp=1689545713828
Jul 17 03:16:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:21.128Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-virtual-net-vethfc45064b.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689545704356
Jul 17 03:16:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:21.145Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_restarts_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", container=\"helm\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-ipmi-exporter-ntwmm\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"6d124c79-91fd-435c-a730-222ebe3b4194\"}" timestamp=1689545713828
Jul 17 03:16:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:21.146Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_user_seconds_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", container=\"telegraf-security\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689545707660
Jul 17 03:16:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:21.179Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_waiting\", asset=\"pcap-tsgx10\", asset_id=\"89\", container=\"helm\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-prometheus-fmhhw\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"4f84a780-549d-4f40-bdec-aee8906d8427\"}" timestamp=1689545713828
Jul 17 03:16:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:21.197Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"80\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545708263
Jul 17 03:16:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:21.212Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"pcap-tsgx10\", asset_id=\"89\", condition=\"true\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"dign-client-g7llw\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"b7bee1cc-245a-40f5-a87e-4563cb9edfda\"}" timestamp=1689545713828
Jul 17 03:16:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:21.229Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_constraint_errors_transmitted_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"mlx5_10\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545708263
Jul 17 03:16:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:21.243Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_info\", address=\"08:c0:eb:53:c9:09\", asset=\"pcap-tsgx10\", asset_id=\"89\", broadcast=\"ff:ff:ff:ff:ff:ff\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens8f1np1\", duplex=\"unknown\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operstate=\"down\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545708263
Jul 17 03:16:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:21.289Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth9b07e674\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545708263
Jul 17 03:16:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:21.300Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_netstat_Udp_InDatagrams\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545708263
Jul 17 03:16:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:21.307Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"33\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545708263
Jul 17 03:16:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:21.342Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"98\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545708263
Jul 17 03:16:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:21.354Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"jvm_memory_max_bytes\", application=\"galaxy-job-executor\", area=\"heap\", asset=\"msh-dellr740-olap002\", asset_id=\"13\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"Galaxy-job-executor-msh-dellr740-olap002\", endpoint_id=\"53\", id=\"PS Old Gen\", module=\"Galaxy-job-executor\", module_id=\"19\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545721271
Jul 17 03:16:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:21.359Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_discards_transmitted_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"mlx5_10\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545708263
Jul 17 03:16:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:21.463Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_device_id\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"br_dign_c\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545708263
Jul 17 03:16:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:21.482Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"phy_rx_packets_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", dev=\"virtio_dign_s\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"mrzcpd-exporter\"}" timestamp=1689545714036
Jul 17 03:16:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:21.499Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"microcode.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"service\"}" timestamp=1689545710657
Jul 17 03:16:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:21.619Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"119\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545708263
Jul 17 03:16:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:21.641Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-update-done.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"oneshot\", type=\"service\"}" timestamp=1689545710657
Jul 17 03:16:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:21.706Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-38e3a560b644b18ec47fddbe282b76d0f18f92096cb0d76cc9e55d5a2b9dad2b.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"scope\"}" timestamp=1689545710657
Jul 17 03:16:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:21.748Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-logind.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"service\"}" timestamp=1689545710657
Jul 17 03:16:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:22.491Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"55\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545708263
Jul 17 03:16:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:22.657Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"jvm_buffer_pool_used_bytes\", asset=\"msh-dellr740-dt003\", asset_id=\"9\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-Yarn-msh-dellr740-dt003-NodeManager\", endpoint_id=\"355\", module=\"DC-Yarn\", module_id=\"1\", nz_agent_id=\"3\", pool=\"direct\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545722405
Jul 17 03:16:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:23.146Z caller=dedupe.go:112 component=remote level=error remote_name=52db6f url=http://10.10.10.159:10090/nz-agent/prometheus/proxy/api/v1/write msg="non-recoverable error" count=1000 exemplarCount=0 err="server returned HTTP status 400 : out of order sample"
Jul 17 03:16:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:23.461Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_last_seen\", asset=\"msh-tsgx09\", asset_id=\"55\", container=\"configmap-reloader\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545718629
Jul 17 03:16:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:23.785Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"rpc-statd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"service\"}" timestamp=1689545718416
Jul 17 03:16:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:23.874Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:a0-0000:a0:03.1-0000:a1:00.0-infiniband-mlx5_6.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689545718416
Jul 17 03:16:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:24.026Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx09\", asset_id=\"55\", container=\"dign-server-web\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Read\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545707363
Jul 17 03:16:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:24.059Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_inodes_free\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/run\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545711491
Jul 17 03:16:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:24.076Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_swap\", asset=\"msh-tsgx09\", asset_id=\"55\", container=\"prometheus-server-configmap-reload\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545711918
Jul 17 03:16:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:24.090Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx09\", asset_id=\"55\", container=\"dign-server-dns\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Discard\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545704609
Jul 17 03:16:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:24.097Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_writes_bytes_total\", asset=\"msh-tsgx09\", asset_id=\"55\", container=\"firewall\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689545713800
Jul 17 03:16:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:24.097Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_spec_replicas\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-1-firewall-6b6bf4679c\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545711124
Jul 17 03:16:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:24.120Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_threads_max\", asset=\"msh-tsgx09\", asset_id=\"55\", container=\"firewall\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689545712281
Jul 17 03:16:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:24.161Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_mcache_sys_bytes\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"ipmi-exporter\"}" timestamp=1689545723120
Jul 17 03:16:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:24.256Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"13\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545714790
Jul 17 03:16:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:24.258Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_working_set_bytes\", asset=\"msh-tsgx09\", asset_id=\"55\", container=\"prometheus-server-configmap-reload\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545711918
Jul 17 03:16:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:24.262Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_compressed_total\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"eno2\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545714790
Jul 17 03:16:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:24.270Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_spec_strategy_rollingupdate_max_surge\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"prometheus-kube-state-metrics\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545720068
Jul 17 03:16:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:24.277Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx09\", asset_id=\"55\", cpu=\"2\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545714790
Jul 17 03:16:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:24.293Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"msh-tsgx09\", asset_id=\"55\", cpu=\"48\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545714790
Jul 17 03:16:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:24.295Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_compressed_total\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens4f0np0\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545714790
Jul 17 03:16:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:24.306Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx09\", asset_id=\"55\", cpu=\"103\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545714790
Jul 17 03:16:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:24.311Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx09\", asset_id=\"55\", container=\"configmap-reloader\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Write\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545716603
Jul 17 03:16:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:24.339Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_constraint_errors_transmitted_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"mlx5_1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545720160
Jul 17 03:16:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:24.356Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_sockstat_TCP_alloc\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545720160
Jul 17 03:16:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:24.367Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx09\", asset_id=\"55\", container=\"dign-server-dns\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Async\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545704609
Jul 17 03:16:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:24.385Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx09\", asset_id=\"55\", cpu=\"24\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545714790
Jul 17 03:16:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:24.391Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-rdma-devices-mlx5_5:1-umad.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689545718416
Jul 17 03:16:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:24.414Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_last_seen\", asset=\"msh-tsgx09\", asset_id=\"55\", container=\"monit-stream\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545718628
Jul 17 03:16:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:24.424Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_packets_total\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"vethba89f19f\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545714790
Jul 17 03:16:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:24.463Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_net_dev_group\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens2f1np1\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545714790
Jul 17 03:16:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:24.524Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_frame_total\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens4f3np3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545714790
Jul 17 03:16:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:24.532Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled_time\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-node-exporter-nxqkm\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"ca423dde-30af-4a5a-99e7-1d03b02ea2b7\"}" timestamp=1689545720068
Jul 17 03:16:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:24.563Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_dormant\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens4f0np0\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545714790
Jul 17 03:16:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:24.637Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"process_open_fds\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545717019
Jul 17 03:16:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:24.654Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_owner\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", owner_is_controller=\"true\", owner_kind=\"DaemonSet\", owner_name=\"prometheus-node-exporter\", pod=\"prometheus-node-exporter-qmjq7\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"e8a13065-a344-44f4-a0e1-7369c47522e2\"}" timestamp=1689545711124
Jul 17 03:16:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:24.654Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_virtual_memory_max_bytes\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"getty@tty1.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689545718416
Jul 17 03:16:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:24.671Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"14\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545720160
Jul 17 03:16:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:24.679Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx09\", asset_id=\"55\", container=\"configmap-reloader\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Read\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545716603
Jul 17 03:16:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:24.681Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"dign-server-dns\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", failure_type=\"pgmajfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"hierarchy\"}" timestamp=1689545723324
Jul 17 03:16:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:24.723Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-405e3b50\\\\x2d5fa3\\\\x2d4f39\\\\x2d9405\\\\x2dac337f946135-volume\\\\x2dsubpaths-telegraf-telegraf-0.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689545718416
Jul 17 03:16:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:24.748Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_info\", asset=\"pcap-tsgx12\", asset_id=\"117\", cluster_ip=\"10.43.100.108\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-alertmanager\", uid=\"974c2df4-39ed-4554-bb1e-5e53ed68f3c2\"}" timestamp=1689545711124
Jul 17 03:16:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:24.759Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"telegraf-security\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", failure_type=\"pgmajfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"hierarchy\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689545719304
Jul 17 03:16:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:24.764Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_spec_type\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-master\", type=\"NodePort\", uid=\"1ab6590d-2e3d-4a1c-bc5a-8fab2a48aa0d\"}" timestamp=1689545711124
Jul 17 03:16:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:24.773Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_net_dev_group\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens7f0np0\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545720160
Jul 17 03:16:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:24.778Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"54\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545720160
Jul 17 03:16:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:24.801Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_annotations\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"tsg-traffic-engine-numa-1-firewall\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545720068
Jul 17 03:16:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:24.810Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_readonly\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"shm\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/930d15af2d3b24fe92be345ef51777909a1badb6f81b9919b3ad0b10b2f7d1b2/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545720160
Jul 17 03:16:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:24.821Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_observed_generation\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-0-firewall-7cd647fdcd\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545711124
Jul 17 03:16:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:24.821Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"39\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545720160
Jul 17 03:16:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:24.842Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"var-lib-kubelet-pods-405e3b50\\\\x2d5fa3\\\\x2d4f39\\\\x2d9405\\\\x2dac337f946135-volume\\\\x2dsubpaths-config\\\\x2dvolume-telegraf\\\\x2dsecurity-2.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689545718416
Jul 17 03:16:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:24.865Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_dormant\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens2f0np0\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545720160
Jul 17 03:16:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:25.019Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_fifo_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"bond0\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545720160
Jul 17 03:16:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:25.540Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_info\", asset=\"pcap-tsgx12\", asset_id=\"117\", configmap=\"sapp-tsg-traffic-engine-numa-0\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545711124
Jul 17 03:16:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:25.558Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_up_changes_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"eno35np2\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545720160
Jul 17 03:16:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:25.869Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_spec_paused\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"prometheus-pushgateway\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545711124
Jul 17 03:16:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:26.057Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_annotations\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"kubernetes-dashboard-66dd5b456\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545718744
Jul 17 03:16:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:26.090Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_excessive_buffer_overrun_errors_total\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_4\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545715810
Jul 17 03:16:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:26.225Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2dpartlabel-EFI\\\\x5cx20System.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689545713907
Jul 17 03:16:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:26.260Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"pcap-kvm-cm001\", asset_id=\"45\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db0\", endpoint=\"CM-Branch-Redis-pcap-kvm-cm001\", endpoint_id=\"321\", host=\"192.168.10.157\", module=\"CM-Branch-Redis\", module_id=\"73\", nz_agent_id=\"3\", parent_asset=\"pcap-dellr740-pve001\", parent_asset_id=\"43\", project=\"Central-Management\", project_id=\"5\"}" timestamp=1689545725365
Jul 17 03:16:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:26.336Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_start_time\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-kube-state-metrics-6b768c7bf8-c5glh\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"1d1df77f-66ac-4156-8ea3-66e8a287970d\"}" timestamp=1689545718744
Jul 17 03:16:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:26.419Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_start_time_seconds\", asset=\"twa-tsgx09\", asset_id=\"107\", container=\"configmap-reloader\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545725490
Jul 17 03:16:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:26.909Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"ftx_bits_total\", app=\"sapp4-0\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"nf_0_fw\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"mrzcpd-exporter\", serviceFunctionIndex=\"0\"}" timestamp=1689545718295
Jul 17 03:16:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:26.925Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_created\", asset=\"twa-tsgx09\", asset_id=\"107\", configmap=\"chart-content-traefik\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545718744
Jul 17 03:16:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:26.944Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"24\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545715810
Jul 17 03:16:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:26.985Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545715810
Jul 17 03:16:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:27.000Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_files_free\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"shm\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/92f30635bf0aef4e1a02a02aa3ee9144ecd7cc53a2abc6f13a3cb185739697c0/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545715810
Jul 17 03:16:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:27.246Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"var-lib-kubelet-pods-880aa12b\\\\x2d51b0\\\\x2d4413\\\\x2d88cb\\\\x2dd06b2fc6d0c3-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2dv8hmc.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689545713907
Jul 17 03:16:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:27.306Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Running\", pod=\"traefik-64f55bb67d-s7cwm\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"80e5751c-f8e9-49bd-ae9e-664fe9b134da\"}" timestamp=1689545718744
Jul 17 03:16:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:27.327Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_spec_strategy_rollingupdate_max_surge\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"configmap-reloader\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545718744
Jul 17 03:16:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:27.359Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"abrt-ccpp.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"service\"}" timestamp=1689545713907
Jul 17 03:16:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:27.430Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_fifo_total\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"virtio_dign_s\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545715810
Jul 17 03:16:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:27.445Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Tcp_Bypass_STM\", app_name=\"sapp4\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", vsysId=\"1\"}" timestamp=1689545715421
Jul 17 03:16:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:27.506Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_waiting\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"telegraf\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", pod=\"tsg-traffic-engine-numa-0-firewall-556f86477-7wv6g\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"0b0292d1-7c6a-4fad-a1f9-f14e3c7cb4f8\"}" timestamp=1689545715682
Jul 17 03:16:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:27.616Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_avail_bytes\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"shm\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/7b13c51d7b16133af270753c151c606633bb81b2611d763e8fbcfbe8be30a5ac/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545722882
Jul 17 03:16:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:27.617Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_heap_sys_bytes\", asset=\"pcap-kvm-cm001\", asset_id=\"45\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"CM-Redis-pcap-kvm-cm001\", endpoint_id=\"305\", host=\"192.168.10.157\", module=\"CM-Redis\", module_id=\"65\", nz_agent_id=\"3\", parent_asset=\"pcap-dellr740-pve001\", parent_asset_id=\"43\", project=\"Central-Management\", project_id=\"5\"}" timestamp=1689545726190
Jul 17 03:16:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:27.635Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_fifo_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"bond0\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545722882
Jul 17 03:16:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:27.658Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"clickhouse_compressed_read_buffer_blocks_total\", asset=\"msh-dellr740-dt003\", asset_id=\"9\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Clickhouse-msh-dellr740-dt003\", endpoint_id=\"75\", module=\"NC-Clickhouse\", module_id=\"29\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545726493
Jul 17 03:16:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:27.820Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db30\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545719446
Jul 17 03:16:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:27.872Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-1f0a130785223ee0108e0cce80d86f231f30900e0a8cc24bfa398555fb8a97aa.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"scope\"}" timestamp=1689545714114
Jul 17 03:16:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:27.892Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"auditd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"service\"}" timestamp=1689545714114
Jul 17 03:16:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:27.934Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2dpartuuid-bf65d95e\\\\x2d6a52\\\\x2d4029\\\\x2da9b6\\\\x2dd4dc7bc235ed.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689545714114
Jul 17 03:16:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:27.980Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_packets_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", interface=\"ens7f0np0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545720168
Jul 17 03:16:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:27.999Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"lb-tcp-80\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Async\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545715013
Jul 17 03:16:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:28.087Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"msh-tsgx10\", asset_id=\"69\", condition=\"Progressing\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"prometheus-pushgateway\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"true\"}" timestamp=1689545715682
Jul 17 03:16:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:28.105Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"117\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545722882
Jul 17 03:16:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:28.126Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_heap_alloc_bytes\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545722882
Jul 17 03:16:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:28.141Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"93\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545722882
Jul 17 03:16:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:28.272Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_compressed_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"vethbd530c93\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545722882
Jul 17 03:16:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:28.635Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_ProcessCallTime_95th_percentile\", asset=\"pcap-dellr740-dt002\", asset_id=\"33\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt002-Hmaster\", endpoint_id=\"345\", module=\"DC-HBase\", module_id=\"3\", name=\"Master\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"IPC\"}" timestamp=1689545727137
Jul 17 03:16:29 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:29.025Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"loki_query_frontend_partitions_bucket\", agent_name=\"msh-kvm-nz001\", application=\"loki\", datacenter=\"MSH\", instance=\"127.0.0.1:13100\", le=\"4.0\"}" timestamp=1689545728327
Jul 17 03:16:29 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:29.135Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"cortex_kv_request_duration_seconds_bucket\", agent_name=\"msh-kvm-nz001\", application=\"loki\", datacenter=\"MSH\", instance=\"127.0.0.1:13100\", kv_name=\"distributor-lifecycler\", le=\"10.0\", operation=\"CAS\", role=\"primary\", status_code=\"200\", type=\"memberlist\"}" timestamp=1689545728327
Jul 17 03:16:29 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:29.220Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"loki_query_frontend_shard_factor_bucket\", agent_name=\"msh-kvm-nz001\", application=\"loki\", datacenter=\"MSH\", instance=\"127.0.0.1:13100\", le=\"16384.0\", mapper=\"shard\"}" timestamp=1689545728327
Jul 17 03:16:30 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:30.935Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"94\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545720939
Jul 17 03:16:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:31.188Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_usage_bytes\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545714245
Jul 17 03:16:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:31.308Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_swap\", asset=\"msh-tsgx05\", asset_id=\"67\", container=\"local-path-provisioner\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545713492
Jul 17 03:16:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:31.619Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-netns-cni\\\\x2d8c227c4e\\\\x2d466e\\\\x2d6393\\\\x2d9a80\\\\x2d5683cf6d08e9.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689545721015
Jul 17 03:16:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:31.731Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_ports\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_endpoint=\"prometheus-kube-state-metrics\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", port_name=\"http\", port_number=\"8080\", port_protocol=\"TCP\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545730436
Jul 17 03:16:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:31.748Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Discard\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689545702390
Jul 17 03:16:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:31.811Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"54\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545719093
Jul 17 03:16:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:31.843Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dracut-pre-trigger.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"service\"}" timestamp=1689545726907
Jul 17 03:16:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:31.878Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-burstable-pod03082888_996f_401e_985d_3fa51721b5b3.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"slice\"}" timestamp=1689545726907
Jul 17 03:16:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:31.919Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"105\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545720791
Jul 17 03:16:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:31.925Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_down_changes_total\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"virtio_dign_c\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545720939
Jul 17 03:16:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:31.948Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_up_changes_total\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens4f2np2\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545720791
Jul 17 03:16:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:31.954Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_commands_duration_seconds_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cmd=\"info\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545730141
Jul 17 03:16:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:32.021Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-870598909289890ab04f058620e78bf132cfba69a9b5efb0acba55a6579c8ca5.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"scope\"}" timestamp=1689545722644
Jul 17 03:16:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:32.149Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"112\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545719093
Jul 17 03:16:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:32.161Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"coredns\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Async\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545710455
Jul 17 03:16:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:32.179Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_restarts_total\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"telegraf-security\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", pod=\"tsg-traffic-engine-numa-1-firewall-6d8d5654d-wdp2k\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"506c9da6-9ad1-4f6d-afc6-e4606512a111\"}" timestamp=1689545719276
Jul 17 03:16:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:32.205Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens4f0np0\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545720791
Jul 17 03:16:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:32.267Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"phy_rx_bits_total\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", dev=\"ens1f1np1\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"mrzcpd-exporter\"}" timestamp=1689545720814
Jul 17 03:16:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:32.309Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"tmpfs\", name=\"run-k3s-containerd-io.containerd.grpc.v1.cri-sandboxes-c3cc78cd8515e4af5b35b37e29a88d93ceee50b536a32cdd4582814be89c2956-shm.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689545726907
Jul 17 03:16:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:32.354Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_speed_bytes\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"vethf7e6d432\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545720791
Jul 17 03:16:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:32.403Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"dashboard-metrics-scraper\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Discard\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545719598
Jul 17 03:16:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:32.436Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_annotations\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"ipmi-exporter\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545719276
Jul 17 03:16:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:32.498Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"openibd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689545722644
Jul 17 03:16:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:32.515Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_file_descriptors\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689545709707
Jul 17 03:16:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:32.589Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"32\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545720605
Jul 17 03:16:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:32.671Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"54\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545720605
Jul 17 03:16:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:32.702Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"94\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545720605
Jul 17 03:16:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:32.717Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"72\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545720605
Jul 17 03:16:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:32.732Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_start_time\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"dign-client-s49j9\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"2096ff4a-92d2-47a7-9b70-293a6568cee4\"}" timestamp=1689545719276
Jul 17 03:16:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:32.744Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_scrape_collector_success\", asset=\"msh-tsgx11\", asset_id=\"111\", collector=\"netdev\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545720791
Jul 17 03:16:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:32.810Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_node_role\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", node=\"localhost\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", role=\"control-plane\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545724068
Jul 17 03:16:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:32.925Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_queue_length\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth171e80b1\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545719093
Jul 17 03:16:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:32.990Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-51d572a6009117bbc0626fe89cf24c538fa97bc90bc5a34e4322f80598516099-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689545727691
Jul 17 03:16:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:33.001Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_reads_total\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda2\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545720648
Jul 17 03:16:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:33.005Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_fifo_total\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethc8144301\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545720605
Jul 17 03:16:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:33.049Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"dign-client\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Discard\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545720907
Jul 17 03:16:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:33.054Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"twa-tsgx03\", asset_id=\"95\", condition=\"unknown\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"configmap-reloader-6c55bbc49f-qmcwt\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"03b47e9e-b9a5-462d-b6ff-6e3865bd5555\"}" timestamp=1689545730436
Jul 17 03:16:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:33.130Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"116\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545720605
Jul 17 03:16:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:33.189Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"83\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545720605
Jul 17 03:16:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:33.211Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"84\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545720605
Jul 17 03:16:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:33.249Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_drop_total\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vetha0e03e9c\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545720605
Jul 17 03:16:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:33.296Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"90\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545720605
Jul 17 03:16:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:33.315Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.grpc.v1.cri-sandboxes-4b26c88a144f1ef458a64110d2d7174f7dd67d4796adb0c6766b9637ba242e5a-shm.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689545726907
Jul 17 03:16:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:33.382Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_info\", asset=\"twa-tsgx05\", asset_id=\"99\", configmap=\"multus-cni-config\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545719894
Jul 17 03:16:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:33.450Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_bytes_total\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens4f2np2\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545719093
Jul 17 03:16:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:33.537Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db225\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545716711
Jul 17 03:16:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:33.609Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-932a67aaa5bdeadf61c2470d22336aefc62897c6245af91f57e5bbb294dba553.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"scope\"}" timestamp=1689545722644
Jul 17 03:16:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:33.629Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"coordinator_segment_size\", asset=\"msh-dellr740-olap001\", asset_id=\"11\", dataSource=\"traffic_general_stat\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Druid-msh-dellr740-olap001\", endpoint_id=\"379\", host=\"192.168.20.221:8081\", module=\"NC-Druid\", module_id=\"31\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", service=\"coordinator\", version=\"0.18.1\"}" timestamp=1689545731911
Jul 17 03:16:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:33.688Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_compressed_total\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth1ccea153\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545720791
Jul 17 03:16:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:33.709Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_packets_total\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"vethd59521a8\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545720791
Jul 17 03:16:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:33.728Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_daemonset_status_desired_number_scheduled\", asset=\"msh-tsgx11\", asset_id=\"111\", daemonset=\"kube-multus-ds\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545719276
Jul 17 03:16:39 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:39.965Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"twa-dellr740-dt003\", asset_id=\"41\", cpu=\"13\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"OLAP-Node-Exporter-twa-dellr740-dt003\", endpoint_id=\"261\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"hbase_master,clickhouse_data,kafka,gohangout,zookeeper,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545738473
Jul 17 03:16:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:40.637Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx04\", asset_id=\"97\", container=\"redis\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Sync\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545720882
Jul 17 03:16:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:40.671Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_file_descriptors\", asset=\"twa-tsgx04\", asset_id=\"97\", container=\"traefik\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545718993
Jul 17 03:16:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:40.775Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_oom_events_total\", asset=\"twa-tsgx04\", asset_id=\"97\", container=\"dign-server-web\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545721631
Jul 17 03:16:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:40.871Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-dellr740-dt003\", asset_id=\"41\", cpu=\"29\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"OLAP-Node-Exporter-twa-dellr740-dt003\", endpoint_id=\"261\", mode=\"nice\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"hbase_master,clickhouse_data,kafka,gohangout,zookeeper,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545738473
Jul 17 03:16:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:41.203Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_storeCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", namespace=\"SYSTEM\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"d7d9af66c3f8401468e797a1d48ebb90\", table=\"CATALOG\", type=\"data\"}" timestamp=1689545739762
Jul 17 03:16:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:41.237Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_storeCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Server\"}" timestamp=1689545739762
Jul 17 03:16:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:41.344Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545731647
Jul 17 03:16:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:41.363Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-traefik-qczmm\", project=\"TSG-OS\", project_id=\"7\", reason=\"NodeAffinity\", service=\"prometheus-kube-state-metrics\", uid=\"719b5e31-40c4-4efe-a513-ed5f56175874\"}" timestamp=1689545740034
Jul 17 03:16:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:41.380Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_mtu_bytes\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"eno36np3\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545731647
Jul 17 03:16:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:41.440Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx06\", asset_id=\"61\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx06\", endpoint_id=\"429\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-sda4.device\", nz_agent_id=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689545738169
Jul 17 03:16:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:41.479Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_appendCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"proxy_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"86027ebeacdd38976ae8d91ec1a3954c\", table=\"index_filename_proxy_hos_bucket\", type=\"index\"}" timestamp=1689545739762
Jul 17 03:16:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:41.527Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_memStoreSize\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"be77ab4255469366247022712c33ee62\", table=\"rtp_hos_bucket\", type=\"data\"}" timestamp=1689545739762
Jul 17 03:16:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:41.586Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_lastMajorCompactionAge\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"c797c1b183075738b860976f05c9f684\", table=\"knowledge_base_hos_bucket\", type=\"data\"}" timestamp=1689545739762
Jul 17 03:16:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:41.593Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx04\", asset_id=\"97\", container=\"dign-server-web\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Read\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545721631
Jul 17 03:16:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:41.628Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_module_fork_in_progress\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545733827
Jul 17 03:16:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:41.644Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db49\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545733827
Jul 17 03:16:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:41.925Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-infiniband-umad2.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689545735764
Jul 17 03:16:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:42.034Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_system_seconds_total\", asset=\"twa-tsgx04\", asset_id=\"97\", container=\"kube-state-metrics\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545717466
Jul 17 03:16:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:42.050Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_dormant\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth4e5d6c5d\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545731647
Jul 17 03:16:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:42.461Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"pcap-tsgx01\", asset_id=\"71\", condition=\"true\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", pod=\"kubernetes-dashboard-66dd5b456-kzhg9\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"3221ac82-9626-479b-b09c-0e5dc5a4acd6\"}" timestamp=1689545735154
Jul 17 03:16:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:42.649Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link_mode\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens6f0np0\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545736242
Jul 17 03:16:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:42.690Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_speed_bytes\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth219b0248\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545731647
Jul 17 03:16:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:42.794Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"46\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545731647
Jul 17 03:16:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:42.817Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_spec_strategy_rollingupdate_max_unavailable\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"metrics-server\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545735154
Jul 17 03:16:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:42.838Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-b99e289a4765b509bc7db04deb128d95fbaa5497abdfb0336f39416a40af0c23.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"scope\"}" timestamp=1689545740671
Jul 17 03:16:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:42.852Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_10_193_1_metric_majorCompactionOutputSize_99th_percentile\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545739762
Jul 17 03:16:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:42.894Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_cpu_seconds_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"NetworkManager.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689545740671
Jul 17 03:16:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:42.897Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_replicas\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-0-firewall-849b856b46\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545735154
Jul 17 03:16:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:42.925Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", cpu=\"59\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545736242
Jul 17 03:16:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:43.019Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_packets_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth0247d195\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545736242
Jul 17 03:16:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:43.059Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_storeCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"session_record_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"6bddb784d97d82b57518e4a1c21b250a\", table=\"index_time_session_record_hos_bucket\", type=\"index\"}" timestamp=1689545739762
Jul 17 03:16:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:43.156Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_sockets\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689545736282
Jul 17 03:16:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:43.190Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_deleteCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"proxy_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"84e88cb916f2401f74e9ca992798ed07\", table=\"index_time_proxy_hos_bucket\", type=\"index\"}" timestamp=1689545739762
Jul 17 03:16:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:43.887Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", cpu=\"122\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545736242
Jul 17 03:16:45 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:45.108Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"73\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545736242
Jul 17 03:16:45 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:45.151Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", cpu=\"10\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545736242
Jul 17 03:16:45 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:45.309Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-systemd-exporter-cbljh\", project=\"TSG-OS\", project_id=\"7\", reason=\"UnexpectedAdmissionError\", service=\"prometheus-kube-state-metrics\", uid=\"da14c78d-f512-4a0f-b73a-808e63c92307\"}" timestamp=1689545735154
Jul 17 03:16:45 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:45.333Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_colls_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens2f1np1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545736242
Jul 17 03:16:45 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:45.371Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_packets_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens7f1np1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545736242
Jul 17 03:16:45 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:45.438Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", cpu=\"67\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545736242
Jul 17 03:16:45 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:45.652Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-ens4f0np0.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689545740671
Jul 17 03:16:47 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:47.522Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"clickhouse_block_discard_bytes_sda\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Clickhouse-msh-dellr740-dt001\", endpoint_id=\"79\", module=\"NC-Clickhouse\", module_id=\"29\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545747154
Jul 17 03:16:48 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:48.924Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_tsg_galaxy_table_gtpc_knowledge_base_metric_deleteTime_median\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545747439
Jul 17 03:16:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:49.503Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_compactionsQueuedCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"session_record_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"3e1c7aa641482ceb8e28b5526f23c5e9\", table=\"index_filename_session_record_hos_bucket\", type=\"index\"}" timestamp=1689545747439
Jul 17 03:16:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:49.548Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"dashboard-metrics-scraper\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Write\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545731334
Jul 17 03:16:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:49.584Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"twa-tsgx10\", asset_id=\"109\", cpu=\"45\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545741797
Jul 17 03:16:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:49.685Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", fstype=\"ext4\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/data/var/lib/kubelet/pods/8bb79bea-895f-4fbf-bba7-ff346d91b1a0/volume-subpaths/config-volume/telegraf-security/2\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545741797
Jul 17 03:16:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:49.687Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-6247313a5016c06851403e117a37ec4c0add7ca2825c21d314257bbe145bf711.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"scope\"}" timestamp=1689545741057
Jul 17 03:16:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:49.721Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"nfs-idmapd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689545741057
Jul 17 03:16:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:49.746Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_4_metric_flushMemstoreSize_min\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545747439
Jul 17 03:16:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:49.801Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"17\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545741797
Jul 17 03:16:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:49.932Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx10\", asset_id=\"109\", cpu=\"111\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545741797
Jul 17 03:16:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:49.961Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-netns-cni\\\\x2dd57ee999\\\\x2d0000\\\\x2db5f0\\\\x2d4f9f\\\\x2d91266601634d.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689545741057
Jul 17 03:16:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:49.989Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_bytes_total\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth9706f3d5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545741797
Jul 17 03:16:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:50.186Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", cpu=\"12\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-olap003\", endpoint_id=\"235\", mode=\"iowait\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"yarn_resourcemanager,zookeeper,nacos,kafka,spark,druid_data,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545749714
Jul 17 03:16:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:50.188Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"druid_index_waiting_task_num\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-olap003\", endpoint_id=\"235\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"yarn_resourcemanager,zookeeper,nacos,kafka,spark,druid_data,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545749714
Jul 17 03:16:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:50.210Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_packets_total\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"em3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-olap003\", endpoint_id=\"235\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"yarn_resourcemanager,zookeeper,nacos,kafka,spark,druid_data,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545749714
Jul 17 03:16:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:50.216Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"rpc-statd-notify.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689545741057
Jul 17 03:16:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:50.248Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_cpu_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-burstable-pode4e2672a_a045_406b_b912_a701b409de63.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"slice\"}" timestamp=1689545746554
Jul 17 03:16:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:50.272Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-sysusers.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689545741057
Jul 17 03:16:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:50.487Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:60-0000:60:03.1-0000:63:00.3-net-eno36np3.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689545741057
Jul 17 03:16:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:50.517Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_up\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"cnibr0\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545746709
Jul 17 03:16:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:50.530Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Unknown_Bit\", app_name=\"sapp4\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", vsysId=\"1\"}" timestamp=1689545748675
Jul 17 03:16:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:50.556Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_usage_seconds_total\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"metrics\", cpu=\"total\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545732035
Jul 17 03:16:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:50.571Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_writes_merged_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"overlay_0-97\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545737629
Jul 17 03:16:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:50.611Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_multicast_total\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"virtio_dign_c\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545741797
Jul 17 03:16:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:50.644Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_mcache_sys_bytes\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"ipmi-exporter\"}" timestamp=1689545739627
Jul 17 03:16:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:50.692Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Udp_Bypass_Pkt\", app_name=\"sapp4\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", vsysId=\"1\"}" timestamp=1689545748675
Jul 17 03:16:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:50.763Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_start_time\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", pod=\"tsg-traffic-engine-numa-1-firewall-58f9975657-n9n7b\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"ba106c44-d5c3-446a-b10b-530b2492f73f\"}" timestamp=1689545739668
Jul 17 03:16:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:50.766Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_frequency_max_hertz\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"52\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545746709
Jul 17 03:16:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:50.832Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_assessment_hos_bucket_metric_avgStoreFileAge\", asset=\"pcap-dellr740-dt002\", asset_id=\"33\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt002-HregionServer\", endpoint_id=\"155\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"1\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545749683
Jul 17 03:16:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:50.851Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"62\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545746709
Jul 17 03:16:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:50.881Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_194_2_metric_flushMemstoreSize_25th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545747439
Jul 17 03:16:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:50.992Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db197\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545746625
Jul 17 03:16:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:51.001Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_writes_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/var/lib/kubelet/pods/59e0fcfb-c5b9-4bb0-aefc-a15fed75746e/volumes/kubernetes.io~projected/kube-api-access-2wnf5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545737629
Jul 17 03:16:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:51.044Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_194_4_metric_scanSize_min\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545747439
Jul 17 03:16:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:51.051Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_deleteCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"assessment_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"94a205daa3421c40aecd15d3f2e84511\", table=\"index_filename_assessment_hos_bucket\", type=\"index\"}" timestamp=1689545747439
Jul 17 03:16:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:51.061Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_scaling_frequency_hertz\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"40\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545746709
Jul 17 03:16:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:51.083Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx10\", asset_id=\"109\", cpu=\"15\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545741797
Jul 17 03:16:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:51.104Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"scrape_samples_scraped\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_instance=\"localhost\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545739050
Jul 17 03:16:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:51.132Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_cpu_seconds_total\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-logind.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689545741057
Jul 17 03:16:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:51.155Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_protocol_type\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"eno35np2\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545746709
Jul 17 03:16:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:51.175Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_cpu_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"irqbalance.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689545746554
Jul 17 03:16:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:51.197Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-d7a1c3193d816621ba701d2ecb26e97e46a53111ef7475343453924603517f21.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"scope\"}" timestamp=1689545741057
Jul 17 03:16:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:51.216Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-br_dign_c.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689545746554
Jul 17 03:16:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:51.242Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx10\", asset_id=\"109\", cpu=\"119\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545741797
Jul 17 03:16:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:51.262Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_errs_total\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"lo\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545741797
Jul 17 03:16:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:51.263Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_maxStoreFileAge\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", namespace=\"SYSTEM\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"bb496e6109e25b411d3e0b58a91e2a0a\", table=\"LOG\", type=\"data\"}" timestamp=1689545747439
Jul 17 03:16:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:51.265Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_blockCacheMissCountPrimary\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Server\"}" timestamp=1689545747439
Jul 17 03:16:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:51.295Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_1_metric_putTime_mean\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545747439
Jul 17 03:16:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:51.306Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_fifo_total\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"bond0\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-olap003\", endpoint_id=\"235\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"yarn_resourcemanager,zookeeper,nacos,kafka,spark,druid_data,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545749714
Jul 17 03:16:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:51.344Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_memory_Dirty_bytes\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-olap003\", endpoint_id=\"235\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"yarn_resourcemanager,zookeeper,nacos,kafka,spark,druid_data,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545749714
Jul 17 03:16:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:51.381Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:00-0000:00:01.1-0000:01:00.0-net-ens1f0np0.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689545746554
Jul 17 03:16:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:51.400Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-b4e8f68416b4183fdf78d6a9674d150c6b831fa6fd738243ec4702238fd5091c.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"scope\"}" timestamp=1689545741057
Jul 17 03:16:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:51.614Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"redis\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Discard\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545735653
Jul 17 03:16:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:51.678Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_spec_replicas\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"prometheus-pushgateway\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545739668
Jul 17 03:16:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:51.690Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_running\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"helm\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-ipmi-exporter-bbkqn\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"af48b541-cb04-486f-951a-b49f9ad44dd5\"}" timestamp=1689545739668
Jul 17 03:16:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:51.729Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_mapped_file\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"dign-server-web\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545735988
Jul 17 03:16:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:51.742Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_labels\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"cm-cache-extra-secrets\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545741568
Jul 17 03:16:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:51.757Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_id\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth767e7158\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545741797
Jul 17 03:16:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:51.790Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_drop_total\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"lo\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545741797
Jul 17 03:16:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:51.821Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_max_fds\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dbus.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689545741057
Jul 17 03:16:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:51.876Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_virtual_memory_bytes\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"udisks2.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689545741057
Jul 17 03:16:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:52.627Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_namespace_status_phase\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", phase=\"Active\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545744478
Jul 17 03:16:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:52.684Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_node_status_allocatable\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", node=\"localhost\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", resource=\"memory\", service=\"prometheus-kube-state-metrics\", unit=\"byte\"}" timestamp=1689545744478
Jul 17 03:16:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:52.803Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-dellr740-dt002\", asset_id=\"33\", cpu=\"38\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"NC-Gohangout-pcap-dellr740-dt002\", endpoint_id=\"173\", mode=\"irq\", module=\"DC-Gohangout\", module_id=\"35\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545752668
Jul 17 03:16:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:53.143Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", container=\"telegraf-security\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", state=\"iowaiting\"}" timestamp=1689545749133
Jul 17 03:16:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:53.214Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"18\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545748328
Jul 17 03:16:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:53.240Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"40\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545748328
Jul 17 03:16:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:53.267Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-infiniband-umad2.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689545745496
Jul 17 03:16:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:53.284Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db119\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545739367
Jul 17 03:16:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:53.287Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-a1690a7c2bcf359cf1475f3f789bc1156fe118be10bfe4e719f29781ee221083.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"scope\"}" timestamp=1689545745496
Jul 17 03:16:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:53.301Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_lookups_total\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"ipmi-exporter\"}" timestamp=1689545739391
Jul 17 03:16:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:53.308Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-virtual-net-veth50cee5b3.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689545745496
Jul 17 03:16:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:53.334Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"process_max_fds\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545748328
Jul 17 03:16:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:53.372Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Succeeded\", pod=\"metrics-server-54dc485875-pqvd7\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"aed8aa53-e603-467c-8ed7-d385c606efb7\"}" timestamp=1689545744478
Jul 17 03:16:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:53.379Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kdump.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689545752135
Jul 17 03:16:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:53.396Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_fifo_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth2169720f\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545749337
Jul 17 03:16:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:53.490Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", cpu=\"37\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545749337
Jul 17 03:16:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:53.504Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx12\", asset_id=\"113\", cpu=\"118\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545743391
Jul 17 03:16:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:53.522Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"msh-tsgx12\", asset_id=\"113\", cpu=\"53\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545743391
Jul 17 03:16:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:53.591Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx08\", asset_id=\"65\", cpu=\"115\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545748328
Jul 17 03:16:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:53.710Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_memory_ShmemPmdMapped_bytes\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545739409
Jul 17 03:16:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:53.727Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_netstat_Udp6_NoPorts\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545739409
Jul 17 03:16:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:53.728Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"94\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545749337
Jul 17 03:16:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:53.737Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", cpu=\"121\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545749337
Jul 17 03:16:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:53.744Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"var-lib-kubelet-pods-0a0a8b78\\\\x2d3fe6\\\\x2d4b35\\\\x2d93e0\\\\x2d3e1bdd94459c-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2drhbn2.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689545740884
Jul 17 03:16:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:53.747Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_vl15_dropped_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"mlx5_6\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545749337
Jul 17 03:16:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:53.784Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"103\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545749337
Jul 17 03:16:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:53.807Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_down_changes_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"eno33np0\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545749337
Jul 17 03:16:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:53.949Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_fully_labeled_replicas\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"dashboard-metrics-scraper-5b94897dc9\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545748950
Jul 17 03:16:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:53.955Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"125\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545749337
Jul 17 03:16:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:53.971Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_mtu_bytes\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"vethc142f50b\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545739409
Jul 17 03:16:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:54.002Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cpu=\"75\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545739409
Jul 17 03:16:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:54.006Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"clevis-luks-askpass.path\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"path\"}" timestamp=1689545752135
Jul 17 03:16:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:54.024Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_info\", address=\"4e:cd:6f:ab:74:ab\", asset=\"pcap-tsgx11\", asset_id=\"115\", broadcast=\"ff:ff:ff:ff:ff:ff\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth736616a4\", duplex=\"full\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operstate=\"up\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545749337
Jul 17 03:16:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:54.042Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", cpu=\"105\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545749337
Jul 17 03:16:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:54.345Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Udp_Bypass_Pkt\", app_name=\"sapp4\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", vsysId=\"1\"}" timestamp=1689545742596
Jul 17 03:16:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:54.363Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_node_status_condition\", asset=\"msh-tsgx12\", asset_id=\"113\", condition=\"DiskPressure\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", node=\"localhost\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"true\"}" timestamp=1689545748663
Jul 17 03:16:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:54.406Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-b6c77c6f732649d45e8a2eb4fce40f2f98156b6d738f96d79069b1e71e9a40a9.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"scope\"}" timestamp=1689545752135
Jul 17 03:16:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:54.462Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"clickhouse_insert_query_total\", asset=\"pcap-dellr740-dt003\", asset_id=\"35\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"NC-Clickhouse-pcap-dellr740-dt003\", endpoint_id=\"85\", module=\"NC-Clickhouse\", module_id=\"29\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545753232
Jul 17 03:16:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:54.480Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"clickhouse_os_irq_time_cpu42\", asset=\"pcap-dellr740-dt003\", asset_id=\"35\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"NC-Clickhouse-pcap-dellr740-dt003\", endpoint_id=\"85\", module=\"NC-Clickhouse\", module_id=\"29\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545753232
Jul 17 03:16:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:54.491Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_physical_state_id\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_4\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545743391
Jul 17 03:16:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:54.514Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_io_time_seconds_total\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/sys/fs/cgroup\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545731127
Jul 17 03:16:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:54.551Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"msh-tsgx12\", asset_id=\"113\", cpu=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545743391
Jul 17 03:16:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:54.554Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_id\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth25a03679\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545748328
Jul 17 03:16:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:54.570Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"msh-tsgx12\", asset_id=\"113\", cpu=\"108\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545743391
Jul 17 03:16:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:54.603Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Read\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689545727946
Jul 17 03:16:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:54.610Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_annotations\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-traefik-crd-ldddg\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"e773b3b3-1ae7-46d9-a705-f7b732c0c435\"}" timestamp=1689545748663
Jul 17 03:16:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:54.614Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_info\", asset=\"msh-tsgx12\", asset_id=\"113\", container=\"helm\", container_id=\"containerd://5ce7ff2cb8a3f298a7682b0c42690bb0fe6db14e51da9dd5e9f8084a72e77ac3\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", image=\"docker.io/rancher/klipper-helm:v0.8.0-build20230510\", image_id=\"sha256:6f42df210d7faa0b6eb56f5f28bb061930c7834af1e7dd4b3586108ec6b1485b\", image_spec=\"rancher/klipper-helm:v0.8.0-build20230510\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-prometheus-k5mgl\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"75952f28-65f8-4ba3-afae-3c693a5ebfb7\"}" timestamp=1689545748663
Jul 17 03:16:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:54.622Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_namespace_status_phase\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", phase=\"Terminating\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545748950
Jul 17 03:16:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:54.712Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"dign-server-2hz4k\", project=\"TSG-OS\", project_id=\"7\", reason=\"UnexpectedAdmissionError\", service=\"prometheus-kube-state-metrics\", uid=\"6a3a1202-966e-4796-80ac-d89a14999091\"}" timestamp=1689545748950
Jul 17 03:16:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:54.883Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_ready_replicas\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"traefik-64f55bb67d\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545742058
Jul 17 03:16:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:54.902Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"0\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545739409
Jul 17 03:16:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:54.946Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_multicast_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"cnibr0\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545739409
Jul 17 03:16:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:54.988Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cpu=\"23\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545739409
Jul 17 03:16:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:56.291Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_annotations\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-0-firewall-687ff5cb59\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545753389
Jul 17 03:16:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:56.397Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_labels\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-1-firewall-6b6bf4679c\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545753389
Jul 17 03:16:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:56.429Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"24\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545745883
Jul 17 03:16:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:56.463Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", container=\"local-path-provisioner\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"cadvisor\", failure_type=\"pgfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"hierarchy\"}" timestamp=1689545747767
Jul 17 03:16:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:56.487Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_usage_bytes\", asset=\"pcap-tsgx03\", asset_id=\"73\", container=\"dashboard-metrics-scraper\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545743563
Jul 17 03:16:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:56.566Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"phy_tx_bits_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", dev=\"ens1f1np1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"mrzcpd-exporter\"}" timestamp=1689545749991
Jul 17 03:16:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:56.694Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", container=\"dashboard-metrics-scraper\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Read\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545743563
Jul 17 03:16:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:56.900Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", container=\"telegraf\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", state=\"running\"}" timestamp=1689545745863
Jul 17 03:16:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:57.013Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"101\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545745883
Jul 17 03:16:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:57.296Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_memory_WritebackTmp_bytes\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545745883
Jul 17 03:16:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:57.325Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"90\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545745883
Jul 17 03:16:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:57.343Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"14\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545745883
Jul 17 03:16:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:57.356Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"tmpfs\", name=\"data-var-lib-kubelet-pods-99474c52\\\\x2d8652\\\\x2d4755\\\\x2db917\\\\x2d62a7ae1a3d6c-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2db2fgz.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689545743372
Jul 17 03:16:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:57.465Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", container=\"configmap-reloader\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Async\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545747012
Jul 17 03:16:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:57.479Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_writes_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", container=\"telegraf\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689545737723
Jul 17 03:16:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:57.484Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_frame_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"eno33np0\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545745883
Jul 17 03:16:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:57.605Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_cpu_shares\", asset=\"pcap-tsgx03\", asset_id=\"73\", container=\"mrapm-device\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545751563
Jul 17 03:16:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:57.610Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"15\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545745883
Jul 17 03:16:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:57.619Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_annotations\", asset=\"pcap-tsgx03\", asset_id=\"73\", configmap=\"sapp-tsg-traffic-engine-numa-0\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545753389
Jul 17 03:16:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:57.626Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_fifo_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens4f3np3\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545745883
Jul 17 03:16:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:57.641Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db187\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545746475
Jul 17 03:16:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:16:57.716Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_info\", asset=\"pcap-tsgx03\", asset_id=\"73\", created_by_kind=\"Job\", created_by_name=\"helm-install-systemd-exporter\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", host_ip=\"10.10.10.163\", host_network=\"false\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", node=\"localhost\", nz_agent_id=\"3\", pod=\"helm-install-systemd-exporter-pw27n\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"6e4c6271-0705-4680-9a24-73bceb8107af\"}" timestamp=1689545753389
Jul 17 03:17:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:00.057Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"msh-tsgx01\", asset_id=\"51\", condition=\"true\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"svclb-traefik-7cfeb2e8-ftd2t\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"fc7f8bf6-1292-4741-9587-65a83ae6dca4\"}" timestamp=1689545747425
Jul 17 03:17:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:00.087Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-kube-state-metrics-6b768c7bf8-fnk7m\", project=\"TSG-OS\", project_id=\"7\", reason=\"UnexpectedAdmissionError\", service=\"prometheus-kube-state-metrics\", uid=\"75a1cd69-272c-4ee3-afeb-f27cbbc02353\"}" timestamp=1689545747425
Jul 17 03:17:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:00.091Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_queue_length\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"bond0\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545749083
Jul 17 03:17:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:00.106Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"53\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545749083
Jul 17 03:17:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:00.222Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_persistentvolume_labels\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", persistentvolume=\"pvc-f9f2ccc6-fa96-4450-b279-c048dfb411cc\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545747425
Jul 17 03:17:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:00.270Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_usage_bytes\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/run/user/1001\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545745422
Jul 17 03:17:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:00.272Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_excessive_buffer_overrun_errors_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_11\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545749083
Jul 17 03:17:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:00.289Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_writes_total\", asset=\"msh-tsgx01\", asset_id=\"51\", container=\"exporter\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545746423
Jul 17 03:17:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:00.302Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"84\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545749083
Jul 17 03:17:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:00.332Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_device_id\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"eno36np3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545749083
Jul 17 03:17:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:00.399Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_colls_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"virtio_dign_c\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545749083
Jul 17 03:17:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:00.421Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_bytes_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"cadvisor\", interface=\"ens2f0np0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545749415
Jul 17 03:17:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:00.472Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_bytes_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens1f1np1\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545749083
Jul 17 03:17:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:00.494Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_frame_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens1f1np1\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545749083
Jul 17 03:17:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:00.596Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"msh-tsgx01\", asset_id=\"51\", container=\"lb-tcp-443\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"cadvisor\", failure_type=\"pgfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"hierarchy\"}" timestamp=1689545743192
Jul 17 03:17:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:00.619Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"plymouth-quit.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"service\"}" timestamp=1689545744879
Jul 17 03:17:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:00.635Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"13\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545749083
Jul 17 03:17:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:00.638Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_oom_events_total\", asset=\"msh-tsgx01\", asset_id=\"51\", container=\"telegraf-security\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689545745424
Jul 17 03:17:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:00.693Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_cpu_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"gssproxy.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689545744879
Jul 17 03:17:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:00.770Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_files_free\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"shm\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/bc21478fdc5355bf15960d43a05074574abd6e1478c55132a193b1b21c029481/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545749083
Jul 17 03:17:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:00.814Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_carrier_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"vethcfad1fbd\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545749083
Jul 17 03:17:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:00.863Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2dpartuuid-e88217ee\\\\x2d754c\\\\x2d46b0\\\\x2d9044\\\\x2d9e0cf7d45857.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689545744879
Jul 17 03:17:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:01.030Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-infiniband-umad5.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689545744879
Jul 17 03:17:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:02.472Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_status_load_balancer_ingress\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", ip=\"10.10.20.162\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"traefik\", uid=\"1a2acd8b-bab5-4447-8944-5478041ae4a8\"}" timestamp=1689545751412
Jul 17 03:17:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:02.513Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_unicast_packets_transmitted_total\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_1\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545758318
Jul 17 03:17:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:02.623Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db131\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545749269
Jul 17 03:17:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:02.858Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx02\", asset_id=\"63\", cpu=\"93\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545758318
Jul 17 03:17:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:02.874Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_forks_total\", asset=\"pcap-dellr740-dt002\", asset_id=\"33\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"OLAP-Node-Exporter-pcap-dellr740-dt002\", endpoint_id=\"253\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545762504
Jul 17 03:17:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:03.125Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"overlay\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-c4efc46b1254571b66c5cc8d21503a79dc3b85733598439da20c2460c3a884f9-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689545748447
Jul 17 03:17:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:03.212Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_state_started\", asset=\"msh-tsgx02\", asset_id=\"63\", container=\"helm\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-traefik-ph7rq\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"eacc1d44-839a-43ee-bf5c-0bccf9558d70\"}" timestamp=1689545751412
Jul 17 03:17:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:03.231Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_bytes_total\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth421cdf56\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545758318
Jul 17 03:17:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:03.609Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_info\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", instance=\"10.10.20.159:10090\", type=\"global\", version=\"go1.19.7\"}" timestamp=1689545762990
Jul 17 03:17:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:04.033Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"em1\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-dt001\", endpoint_id=\"225\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545762988
Jul 17 03:17:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:04.083Z caller=dedupe.go:112 component=remote level=info remote_name=52db6f url=http://10.10.10.159:10090/nz-agent/prometheus/proxy/api/v1/write msg="Dropped sample for series that was not explicitly dropped via relabelling" ref=4436650
Jul 17 03:17:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:04.128Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"msh-tsgx02\", asset_id=\"63\", condition=\"unknown\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-ipmi-exporter-mqzww\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"d3f976a5-a42f-4d79-8f22-dcd41aaf53f6\"}" timestamp=1689545751412
Jul 17 03:17:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:07.311Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"cortex_blocks_meta_sync_duration_seconds_bucket\", agent_name=\"msh-kvm-nz001\", application=\"cortex\", component=\"querier\", datacenter=\"MSH\", instance=\"127.0.0.1:10093\", le=\"+Inf\"}" timestamp=1689545767263
Jul 17 03:17:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:07.874Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"cortex_request_message_bytes_bucket\", agent_name=\"msh-kvm-nz001\", application=\"cortex\", datacenter=\"MSH\", instance=\"127.0.0.1:10093\", le=\"1.048576e+06\", method=\"gRPC\", route=\"/frontend.Frontend/Process\"}" timestamp=1689545767263
Jul 17 03:17:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:07.913Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"twa-tsgx01\", asset_id=\"91\", container=\"lb-tcp-80\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"running\"}" timestamp=1689545760274
Jul 17 03:17:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:07.928Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_info\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_endpoint=\"kubernetes-dashboard\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545761227
Jul 17 03:17:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:08.039Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_up_changes_total\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth2d0ed10f\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545763910
Jul 17 03:17:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:08.050Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cpu=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545763910
Jul 17 03:17:09 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:09.943Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_putCount\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"default\", project=\"OLAP\", project_id=\"3\", region=\"8fdbb0afc100cef71526519ab87bf161\", table=\"index_partfile_default\", type=\"index\"}" timestamp=1689545768805
Jul 17 03:17:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:10.889Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_replicaid\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"firewall_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"cde0c9c691b14de857d83961ff8e99c8\", table=\"index_partfile_firewall_hos_bucket\", type=\"index\"}" timestamp=1689545768805
Jul 17 03:17:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:11.140Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_avgStoreFileAge\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", module=\"DC-HBase\", module_id=\"3\", namespace=\"SYSTEM\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"99211ccab1648d1974131d70ea51a2d7\", table=\"LOG\", type=\"data\"}" timestamp=1689545768805
Jul 17 03:17:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:11.317Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_1_metric_flushMemstoreSize_median\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545770147
Jul 17 03:17:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:11.973Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_numReferenceFiles\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"rtp_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"afda67220314ac681ee8f19c890bcc36\", table=\"index_partfile_rtp_hos_bucket\", type=\"index\"}" timestamp=1689545770147
Jul 17 03:17:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:11.996Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_1_metric_flushMemstoreSize_99_9th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545770147
Jul 17 03:17:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:12.118Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_4_metric_majorCompactionInputFileCount_99_9th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545770147
Jul 17 03:17:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:12.295Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_storeCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"troubleshooting_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"61e1e1786af00299f043aca02228e889\", table=\"index_filename_troubleshooting_hos_bucket\", type=\"index\"}" timestamp=1689545770147
Jul 17 03:17:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:12.298Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_193_1_metric_majorCompactionOutputFileCount_median\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545770147
Jul 17 03:17:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:12.311Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_lastMajorCompactionAge\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"proxy_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"7bcf63599704330b114c0c7e0332621c\", table=\"index_time_proxy_hos_bucket\", type=\"index\"}" timestamp=1689545770147
Jul 17 03:17:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:12.321Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_LOG_metric_deleteBatchTime_num_ops\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545770147
Jul 17 03:17:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:12.577Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_TASK_metric_putBatchTime_90th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545770147
Jul 17 03:17:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:12.608Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_numFilesCompactedCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"troubleshooting_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"b3f77aa4a62fb97879f86a184bd53a12\", table=\"index_partfile_troubleshooting_hos_bucket\", type=\"index\"}" timestamp=1689545770147
Jul 17 03:17:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:12.870Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_default_metric_compactionOutputSize_99th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545770147
Jul 17 03:17:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:12.888Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_getCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"firewall_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"54c245281207039aea952a7c7045fe85\", table=\"index_partfile_firewall_hos_bucket\", type=\"index\"}" timestamp=1689545770147
Jul 17 03:17:13 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:13.300Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_incrementCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"49caac8a08a11079694f8ae38226a71f\", table=\"session_record_hos_bucket\", type=\"data\"}" timestamp=1689545770147
Jul 17 03:17:13 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:13.315Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_194_2_metric_majorCompactionTime_median\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545770147
Jul 17 03:17:13 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:13.316Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_compactionsFailedCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"2edbbe7283cdae3080366445582b7dac\", table=\"rtp_hos_bucket\", type=\"data\"}" timestamp=1689545770147
Jul 17 03:17:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:15.424Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db160\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545766027
Jul 17 03:17:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:15.894Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_net_dev_group\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth507b7852\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545761230
Jul 17 03:17:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:15.924Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_bytes_total\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth889a735f\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545761230
Jul 17 03:17:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:15.971Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_vl15_dropped_total\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_6\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545761230
Jul 17 03:17:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:16.106Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Udp_Link_Double\", app_name=\"sapp4\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", vsysId=\"1\"}" timestamp=1689545760513
Jul 17 03:17:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:16.247Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-netns-cni\\\\x2d6d710027\\\\x2d0dd6\\\\x2d735d\\\\x2d7057\\\\x2d02bb5a6bbf86.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689545767046
Jul 17 03:17:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:16.255Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_size_bytes\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"shm\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/2c1282694572bf22945539738d3e76ea74b899f7d6d046e083c67d2aab230f0f/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545761230
Jul 17 03:17:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:16.273Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_device_id\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethe34782c1\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545761230
Jul 17 03:17:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:16.362Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"58\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545761230
Jul 17 03:17:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:16.376Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_net_dev_group\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens4f0np0\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545761230
Jul 17 03:17:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:16.402Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2dpartlabel-TSG\\\\x2dOS\\\\x2dDATA.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689545767046
Jul 17 03:17:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:16.460Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"nfs-mountd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689545767046
Jul 17 03:17:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:16.609Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_restarts_total\", asset=\"twa-tsgx07\", asset_id=\"103\", container=\"exporter-merger\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"mrzcpd-exporter-5cf7f967cc-95k2q\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"a324e348-d97e-48ff-833e-cccb478bc4df\"}" timestamp=1689545768076
Jul 17 03:17:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:16.631Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"79\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545761230
Jul 17 03:17:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:16.646Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_data_transmitted_bytes_total\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545761230
Jul 17 03:17:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:16.735Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db165\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545766027
Jul 17 03:17:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:16.839Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_status_failed\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", job_name=\"helm-install-systemd-exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545768076
Jul 17 03:17:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:16.864Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_state_started\", asset=\"twa-tsgx07\", asset_id=\"103\", container=\"local-path-provisioner\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"local-path-provisioner-957fdf8bc-xbs2n\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"eefa9388-f142-43b8-b914-362041946ff8\"}" timestamp=1689545768076
Jul 17 03:17:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:16.877Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_file_descriptors\", asset=\"twa-tsgx07\", asset_id=\"103\", container=\"dign-client\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545759759
Jul 17 03:17:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:16.914Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db49\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545766027
Jul 17 03:17:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:16.924Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_commands_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cmd=\"select\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545766027
Jul 17 03:17:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:16.950Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_writes_total\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689545754974
Jul 17 03:17:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:16.969Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"twa-tsgx07\", asset_id=\"103\", container=\"lb-tcp-80\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"iowaiting\"}" timestamp=1689545758853
Jul 17 03:17:17 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:17.040Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_ips\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", ip=\"10.42.0.55\", ip_family=\"4\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"metrics-server-54dc485875-cw4sf\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"a203ebf3-8366-4cfd-b036-168e793673ce\"}" timestamp=1689545768076
Jul 17 03:17:17 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:17.053Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_bytes_total\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"cadvisor\", interface=\"ens2f0np0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545767124
Jul 17 03:17:17 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:17.067Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_reservation_limit_bytes\", asset=\"twa-tsgx07\", asset_id=\"103\", container=\"kube-multus\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545769150
Jul 17 03:17:17 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:17.074Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-tsg-traffic-engine-numa-1-xzrjz\", project=\"TSG-OS\", project_id=\"7\", reason=\"UnexpectedAdmissionError\", service=\"prometheus-kube-state-metrics\", uid=\"1a99348c-8f01-4b44-9d18-75fe79532073\"}" timestamp=1689545768076
Jul 17 03:17:17 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:17.112Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"shm\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/9fd5698ff2e53618e024f2212ebedf94fe287a8a8af788944c8f9ccdc69f33de/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545761230
Jul 17 03:17:17 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:17.139Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545761230
Jul 17 03:17:17 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:17.146Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"79\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545761230
Jul 17 03:17:17 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:17.158Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_cpu_seconds_total\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-poda8ca687f_46d5_4647_ba3b_841b55595ad2.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"slice\"}" timestamp=1689545767046
Jul 17 03:17:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:19.408Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-ens2f0np0.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689545764356
Jul 17 03:17:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:19.434Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"108\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545777612
Jul 17 03:17:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:19.517Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_waiting\", asset=\"twa-tsgx06\", asset_id=\"101\", container=\"prometheus-server\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-server-6ff784d758-zzlsk\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"9041122f-12ad-44d4-a492-63eb077aa8d8\"}" timestamp=1689545773949
Jul 17 03:17:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:19.581Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_cpu_seconds_total\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"k3s.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689545764356
Jul 17 03:17:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:19.595Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.grpc.v1.cri-sandboxes-ed6f0ea82150414c9db1c231ca8955c25471a932664823fafe082c7717c8f5f7-shm.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689545764356
Jul 17 03:17:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:19.607Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_resident_memory_bytes\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"chronyd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689545764356
Jul 17 03:17:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:19.649Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-sda4.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689545764356
Jul 17 03:17:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:19.694Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"tmpfs\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/data/var/lib/kubelet/pods/5efb71d9-5988-4e2b-a058-7799e16a5dfa/volumes/kubernetes.io~projected/kube-api-access-rl7g8\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545777612
Jul 17 03:17:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:19.961Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"twa-tsgx06\", asset_id=\"101\", container=\"prometheus-node-exporter\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"sleeping\"}" timestamp=1689545760889
Jul 17 03:17:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:19.991Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_info\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"kubernetes-dashboard-csrf\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545773949
Jul 17 03:17:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:20.061Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-5f8b23e9\\\\x2d8830\\\\x2d4b0a\\\\x2d9d61\\\\x2d06b46edd96f0-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2dwzqnt.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689545764356
Jul 17 03:17:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:20.081Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_mapped_file\", asset=\"twa-tsgx06\", asset_id=\"101\", container=\"monit-stream\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545766393
Jul 17 03:17:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:20.094Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_gc_duration_seconds_count\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545777612
Jul 17 03:17:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:20.109Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"45\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545777612
Jul 17 03:17:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:20.122Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_rss\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689545768120
Jul 17 03:17:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:20.240Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"loadmodules.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"service\"}" timestamp=1689545764356
Jul 17 03:17:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:20.310Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-e9640b22e777a73f08ac4b527a08ca4d205bdf496911c2046a76b9fd43977450-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689545764356
Jul 17 03:17:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:20.327Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_max_usage_bytes\", asset=\"pcap-tsgx10\", asset_id=\"89\", container=\"prometheus-server\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545766034
Jul 17 03:17:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:20.393Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_down_changes_total\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens3f1np1\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545777612
Jul 17 03:17:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:20.544Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"plymouth-quit-wait.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"service\"}" timestamp=1689545770657
Jul 17 03:17:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:20.573Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"5\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545777612
Jul 17 03:17:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:20.590Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_edac_csrow_uncorrectable_errors_total\", asset=\"twa-tsgx06\", asset_id=\"101\", controller=\"1\", csrow=\"unknown\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545777612
Jul 17 03:17:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:20.677Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_bytes_total\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"cnibr0\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545777612
Jul 17 03:17:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:20.754Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"17\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545768263
Jul 17 03:17:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:20.791Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_owner\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", owner_is_controller=\"true\", owner_kind=\"ReplicaSet\", owner_name=\"tsg-traffic-engine-numa-0-firewall-5db5dbc44c\", pod=\"tsg-traffic-engine-numa-0-firewall-5db5dbc44c-866zl\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"9145d365-24c7-4ff1-a1ee-0a800b33502b\"}" timestamp=1689545773949
Jul 17 03:17:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:20.812Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_errors_received_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"mlx5_4\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545768263
Jul 17 03:17:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:20.829Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_drop_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth8d94c437\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545768263
Jul 17 03:17:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:20.895Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_annotations\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_endpoint=\"cm-policy-local-cache-redis-headless\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545773949
Jul 17 03:17:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:21.276Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"29\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545768263
Jul 17 03:17:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:21.329Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_restart_policy\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"kube-multus-ds-sqrb7\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", type=\"Always\", uid=\"ee8afe42-b7fb-4840-a9ee-3d0e7212c356\"}" timestamp=1689545773828
Jul 17 03:17:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:21.351Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_type\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.tsg-traffic-engine-numa-1.v3\", service=\"prometheus-kube-state-metrics\", type=\"helm.sh/release.v1\"}" timestamp=1689545773828
Jul 17 03:17:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:21.510Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545768263
Jul 17 03:17:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:22.247Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link\", asset=\"msh-tsgx07\", asset_id=\"53\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens8f0np0\", endpoint=\"TSG-OS-msh-tsgx07\", endpoint_id=\"421\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545776678
Jul 17 03:17:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:22.392Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_hwmon_temp_crit_celsius\", asset=\"msh-dellr740-olap004\", asset_id=\"17\", chip=\"platform_coretemp_1\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Spark-msh-dellr740-olap004\", endpoint_id=\"15\", module=\"NC-Spark\", module_id=\"47\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sensor=\"temp5\"}" timestamp=1689545782255
Jul 17 03:17:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:22.654Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"OLAP-Node-Exporter-twa-dellr740-dt001\", endpoint_id=\"257\", module=\"OLAP-Node-Exporter\", module_id=\"51\", name=\"47\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\", type=\"Processor\"}" timestamp=1689545781405
Jul 17 03:17:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:22.810Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_fifo_total\", asset=\"msh-dellr740-olap004\", asset_id=\"17\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"em2\", endpoint=\"NC-Spark-msh-dellr740-olap004\", endpoint_id=\"15\", module=\"NC-Spark\", module_id=\"47\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545782255
Jul 17 03:17:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:23.459Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-virtual-net-vethee323c11.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689545778416
Jul 17 03:17:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:23.476Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_daemonset_created\", asset=\"msh-tsgx09\", asset_id=\"55\", daemonset=\"dign-client\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545780068
Jul 17 03:17:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:23.499Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"overlay\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-0bf21c684b5dc9877d9ee0eac226b483f9fccbc9cba592f7e5f290343873248a-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689545778416
Jul 17 03:17:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:23.530Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_packets_total\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", interface=\"br_dign_c\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545774176
Jul 17 03:17:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:23.541Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_status_failed\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", job_name=\"helm-install-tsg-traffic-engine-numa-0\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545780068
Jul 17 03:17:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:23.562Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"msh-tsgx09\", asset_id=\"55\", condition=\"false\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"svclb-traefik-53c779fd-wwnxx\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"17a39b41-f716-4eab-a979-0f0f67f99785\"}" timestamp=1689545780068
Jul 17 03:17:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:23.619Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"auth-rpcgss-module.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"service\"}" timestamp=1689545778416
Jul 17 03:17:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:23.682Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_usage_bytes\", asset=\"msh-tsgx09\", asset_id=\"55\", container=\"prometheus-pushgateway\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545760523
Jul 17 03:17:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:23.686Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_frame_total\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"eno33np0\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545774790
Jul 17 03:17:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:23.693Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_cpu_shares\", asset=\"msh-tsgx09\", asset_id=\"55\", container=\"prometheus-server\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545778586
Jul 17 03:17:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:23.799Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"msh-tsgx09\", asset_id=\"55\", cpu=\"114\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545774790
Jul 17 03:17:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:23.865Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx09\", asset_id=\"55\", cpu=\"50\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545774790
Jul 17 03:17:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:23.876Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-e29ba7f3c6e483ee504e55d39207f8bc66701a3ab2cd81ede609c6ee5b1d703d-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689545778416
Jul 17 03:17:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:23.987Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_data_transmitted_bytes_total\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_10\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545774790
Jul 17 03:17:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:24.039Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-kernel-config.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689545778416
Jul 17 03:17:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:24.040Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_device_id\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"virtio_dign_s\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545774790
Jul 17 03:17:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:24.058Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-0632ec2f152e4f21e124df9c303633ee127dc67fc3bcb7eab93790c3f911033a.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"scope\"}" timestamp=1689545778416
Jul 17 03:17:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:24.125Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"msh-tsgx09\", asset_id=\"55\", condition=\"Progressing\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"local-path-provisioner\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"false\"}" timestamp=1689545780068
Jul 17 03:17:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:24.136Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"system.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"slice\"}" timestamp=1689545778416
Jul 17 03:17:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:24.171Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dbus.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"service\"}" timestamp=1689545778416
Jul 17 03:17:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:24.204Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_start_time_seconds\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"kubernetes-dashboard\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545783419
Jul 17 03:17:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:24.231Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_cache\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"prometheus-server\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545772437
Jul 17 03:17:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:24.245Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_ready\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"dashboard-metrics-scraper\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", pod=\"dashboard-metrics-scraper-5b94897dc9-hgw87\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"b65f87c9-c665-4fc6-9a65-8ece7ccf5f5a\"}" timestamp=1689545771122
Jul 17 03:17:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:24.286Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_queue_length\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"eno33np0\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545780160
Jul 17 03:17:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:24.348Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx09\", asset_id=\"55\", container=\"dign-client\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Total\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545774270
Jul 17 03:17:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:24.363Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_bytes_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens4f3np3\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545780160
Jul 17 03:17:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:24.381Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"80\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545780160
Jul 17 03:17:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:24.488Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_sockets\", asset=\"msh-tsgx09\", asset_id=\"55\", container=\"dashboard-metrics-scraper\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545765179
Jul 17 03:17:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:24.684Z caller=dedupe.go:112 component=remote level=error remote_name=52db6f url=http://10.10.10.159:10090/nz-agent/prometheus/proxy/api/v1/write msg="non-recoverable error" count=1000 exemplarCount=0 err="server returned HTTP status 400 : out of order sample"
Jul 17 03:17:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:24.700Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"msh-tsgx09\", asset_id=\"55\", condition=\"unknown\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-configmap-reloader-84s59\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"47ba69e1-fd09-4e39-941a-89000685484e\"}" timestamp=1689545780068
Jul 17 03:17:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:24.733Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db94\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545777019
Jul 17 03:17:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:24.754Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-netns-cni\\\\x2d3e56315b\\\\x2d1c94\\\\x2d09f1\\\\x2dd22b\\\\x2dfc410e7a2777.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689545778416
Jul 17 03:17:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:24.861Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"local-fs-pre.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"target\"}" timestamp=1689545778416
Jul 17 03:17:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:25.128Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_usage_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"redis\", cpu=\"total\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545780445
Jul 17 03:17:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:25.264Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"swap.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"target\"}" timestamp=1689545778416
Jul 17 03:17:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:25.412Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"iscsiuio.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689545771102
Jul 17 03:17:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:25.430Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_last_seen\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"ipmi-exporter\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545783448
Jul 17 03:17:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:25.430Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"rsyslog.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689545771102
Jul 17 03:17:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:25.512Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_packets_dropped_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", interface=\"ens1f0np0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545783129
Jul 17 03:17:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:25.675Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"95\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545780160
Jul 17 03:17:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:25.695Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kafka_network_RequestMetrics_Errors_total\", asset=\"msh-dellr740-dt003\", asset_id=\"9\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-Kafka-msh-dellr740-dt003\", endpoint_id=\"97\", error=\"NONE\", module=\"DC-Kafka\", module_id=\"5\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", request=\"OffsetFetch\"}" timestamp=1689545784884
Jul 17 03:17:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:25.715Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"54\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545780160
Jul 17 03:17:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:25.731Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_tasks_current\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-tmpfiles-setup.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689545771102
Jul 17 03:17:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:25.879Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_persistentvolume_info\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", persistentvolume=\"pvc-06f1fc97-f004-48b3-97d5-b2c7e3231e2b\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", storageclass=\"local-path\"}" timestamp=1689545771122
Jul 17 03:17:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:25.893Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_arp_entries\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"cnibr0\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545780160
Jul 17 03:17:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:25.911Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_scrape_collector_duration_seconds\", asset=\"pcap-tsgx12\", asset_id=\"117\", collector=\"ipvs\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545780160
Jul 17 03:17:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:25.911Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_drop_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"eno33np0\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545780160
Jul 17 03:17:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:26.030Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_multicast_packets_received_total\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_8\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545775810
Jul 17 03:17:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:26.107Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_entropy_pool_size_bits\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545775810
Jul 17 03:17:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:26.169Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"clickhouse_os_system_time_cpu9\", asset=\"twa-dellr740-dt002\", asset_id=\"39\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"NC-Clickhouse-twa-dellr740-dt002\", endpoint_id=\"89\", module=\"NC-Clickhouse\", module_id=\"29\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545784866
Jul 17 03:17:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:26.285Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"abrt-journal-core.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"simple\", type=\"service\"}" timestamp=1689545773907
Jul 17 03:17:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:26.319Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_bytes_total\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethb8a1e98e\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545775810
Jul 17 03:17:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:26.322Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_net_dev_group\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens2f1np1\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545775810
Jul 17 03:17:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:26.365Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"3\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545775810
Jul 17 03:17:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:26.420Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_scrape_collector_success\", asset=\"twa-tsgx09\", asset_id=\"107\", collector=\"time\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545775810
Jul 17 03:17:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:26.685Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_metadata_generation\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-0-firewall-59d47d489\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545778744
Jul 17 03:17:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:26.705Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"41\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545775810
Jul 17 03:17:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:26.808Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"49\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545775810
Jul 17 03:17:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:26.808Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"56\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545775810
Jul 17 03:17:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:26.822Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-16605650\\\\x2dcdf7\\\\x2d400d\\\\x2d9016\\\\x2d8c2dbe1159c9-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2d62msv.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689545773907
Jul 17 03:17:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:26.834Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"117\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545775810
Jul 17 03:17:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:26.896Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"twa-tsgx09\", asset_id=\"107\", condition=\"true\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"metrics-server-54dc485875-24t2d\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"25066531-7d86-4be0-be75-7e9052465c16\"}" timestamp=1689545778744
Jul 17 03:17:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:26.914Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_scrape_collector_success\", asset=\"twa-tsgx09\", asset_id=\"107\", collector=\"powersupplyclass\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545775810
Jul 17 03:17:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:26.935Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db235\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545783549
Jul 17 03:17:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:26.953Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_cpu_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-pod880aa12b_51b0_4413_88cb_d06b2fc6d0c3.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"slice\"}" timestamp=1689545773907
Jul 17 03:17:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:26.973Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"twa-tsgx09\", asset_id=\"107\", container=\"exporter\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"cadvisor\", failure_type=\"pgfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"hierarchy\"}" timestamp=1689545784825
Jul 17 03:17:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:27.098Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"77\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545775810
Jul 17 03:17:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:27.109Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2did-wwn\\\\x2d0x62cea7f08346b9002c25f712ca485ab5\\\\x2dpart5.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689545773907
Jul 17 03:17:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:27.142Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-virtual-net-veth4c7179cd.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689545773907
Jul 17 03:17:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:27.239Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"43\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545775810
Jul 17 03:17:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:27.558Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"29\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545782882
Jul 17 03:17:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:27.595Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_last_gc_time_seconds\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545779446
Jul 17 03:17:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:28.012Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"53\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545782882
Jul 17 03:17:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:28.475Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"dign-client\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Discard\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545778991
Jul 17 03:17:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:28.741Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_QueueCallTime_90th_percentile\", asset=\"pcap-dellr740-dt002\", asset_id=\"33\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt002-Hmaster\", endpoint_id=\"345\", module=\"DC-HBase\", module_id=\"3\", name=\"Master\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"IPC\"}" timestamp=1689545787137
Jul 17 03:17:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:28.836Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_node_info\", asset=\"msh-tsgx10\", asset_id=\"69\", container_runtime_version=\"containerd://1.7.1-k3s1\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", internal_ip=\"10.10.20.170\", kernel_version=\"5.17.15-1.el8.x86_64\", kubelet_version=\"v1.27.2+k3s1\", kubeproxy_version=\"v1.27.2+k3s1\", module=\"TSG-OS\", module_id=\"77\", node=\"localhost\", nz_agent_id=\"3\", os_image=\"TSG-OS v23.07-rc1-079e964 (TSGXNXR620G40R01P0906)\", pod_cidr=\"10.42.0.0/24\", project=\"TSG-OS\", project_id=\"7\", provider_id=\"k3s://localhost\", service=\"prometheus-kube-state-metrics\", system_uuid=\"4c4c4544-0046-5410-8038-cac04f324733\"}" timestamp=1689545775682
Jul 17 03:17:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:28.900Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_annotations\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-mrzcpd-exporter-rqnc6\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"618f1374-5d25-487d-a309-56dfc1b40ce1\"}" timestamp=1689545775682
Jul 17 03:17:29 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:29.463Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_multicast_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens8f1np1\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545782882
Jul 17 03:17:29 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:29.888Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"loki_ingester_chunk_utilization_count\", agent_name=\"msh-kvm-nz001\", application=\"loki\", datacenter=\"MSH\", instance=\"127.0.0.1:13100\"}" timestamp=1689545788328
Jul 17 03:17:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:31.524Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_bytes_total\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth914fec83\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545779093
Jul 17 03:17:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:31.629Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_scrape_collector_success\", asset=\"twa-tsgx03\", asset_id=\"95\", collector=\"uname\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545779093
Jul 17 03:17:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:31.666Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_spec_strategy_rollingupdate_max_surge\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"prometheus-server\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545790436
Jul 17 03:17:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:31.685Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"dign-server-t24gc\", project=\"TSG-OS\", project_id=\"7\", reason=\"UnexpectedAdmissionError\", service=\"prometheus-kube-state-metrics\", uid=\"1d72bbea-870a-432c-b71c-26fca67b7d6c\"}" timestamp=1689545784068
Jul 17 03:17:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:31.795Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_metadata_resource_version\", asset=\"msh-tsgx05\", asset_id=\"67\", configmap=\"chart-content-configmap-reloader\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545784068
Jul 17 03:17:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:31.956Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"14\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545780939
Jul 17 03:17:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:32.024Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_labels\", asset=\"msh-tsgx05\", asset_id=\"67\", configmap=\"cm-cache-extra-scripts\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545784068
Jul 17 03:17:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:32.045Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"13\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545779093
Jul 17 03:17:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:32.046Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_restarts_total\", asset=\"msh-tsgx05\", asset_id=\"67\", container=\"dign-client\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"dign-client-469mx\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"9aea822c-d5bb-4081-ad0f-a7aa2fc38dfc\"}" timestamp=1689545784068
Jul 17 03:17:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:32.067Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"51\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545780939
Jul 17 03:17:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:32.098Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_constraint_errors_transmitted_total\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_1\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545779093
Jul 17 03:17:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:32.116Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-16b0c54f99bc831e5c2f40ec71018ed8402840a09fc9afa8e373fbc9e8816ba5-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689545781015
Jul 17 03:17:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:32.122Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_net_dev_group\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth5cd72b5c\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545779093
Jul 17 03:17:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:32.129Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_readonly\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"shm\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/ee64a84f3a91bcf69233b3b80282c63a9e32c3354ec84765157c122c18f22c96/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545780939
Jul 17 03:17:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:32.155Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_drop_total\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth9a152f90\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545780791
Jul 17 03:17:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:32.179Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"33\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545780791
Jul 17 03:17:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:32.199Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_mtu_bytes\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"br_dign_c\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545780791
Jul 17 03:17:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:32.226Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"105\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545780791
Jul 17 03:17:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:32.230Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"lb-tcp-80\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", failure_type=\"pgmajfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"hierarchy\"}" timestamp=1689545772882
Jul 17 03:17:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:32.232Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"prometheus-kube-state-metrics\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545779276
Jul 17 03:17:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:32.250Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_annotations\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-server-6ff784d758-mv9fn\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"461d1588-9ea2-41cf-8eca-daa78ce6d555\"}" timestamp=1689545779276
Jul 17 03:17:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:32.258Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-node-exporter-zwvkk\", project=\"TSG-OS\", project_id=\"7\", reason=\"NodeLost\", service=\"prometheus-kube-state-metrics\", uid=\"677536dd-0637-4a15-9edc-d65a4d03d2c3\"}" timestamp=1689545784068
Jul 17 03:17:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:32.298Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda5\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", fstype=\"ext4\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/var/lib/kubelet/pods/abf27d03-a067-448d-81a8-11df769a2208/volume-subpaths/config-volume/firewall/1\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545780791
Jul 17 03:17:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:32.363Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_ips\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", ip=\"10.42.0.102\", ip_family=\"4\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", pod=\"kubernetes-dashboard-66dd5b456-kp8kd\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"b2856aaa-c422-4282-be75-d8cf7ebd4f92\"}" timestamp=1689545790436
Jul 17 03:17:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:32.407Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_mspan_inuse_bytes\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"ipmi-exporter\"}" timestamp=1689545783498
Jul 17 03:17:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:32.420Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"multi-user.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"target\"}" timestamp=1689545781015
Jul 17 03:17:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:32.433Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-cb28165987f20fc2f15a6f64f58b8c4c01aa0933e9cad3d241eba2bc6ab3c468-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689545781015
Jul 17 03:17:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:32.454Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_owner\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", owner_is_controller=\"true\", owner_kind=\"Deployment\", owner_name=\"configmap-reloader\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"configmap-reloader-6c55bbc49f\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545779276
Jul 17 03:17:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:32.455Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_info\", asset=\"twa-tsgx05\", asset_id=\"99\", container=\"kubernetes-dashboard\", container_id=\"containerd://e93867ae88f0636faa4500428f8acafcdb4696607b981a863305749e52ede700\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", image=\"docker.io/kubernetesui/dashboard:v2.7.0\", image_id=\"sha256:07655ddf2eebe5d250f7a72c25f638b27126805d61779741b4e62e69ba080558\", image_spec=\"docker.io/kubernetesui/dashboard:v2.7.0\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", pod=\"kubernetes-dashboard-66dd5b456-nqg8s\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"e02c929a-3c07-4477-b04a-8ffc16c405f5\"}" timestamp=1689545779894
Jul 17 03:17:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:32.471Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_carrier_total\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens4f0np0\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545780605
Jul 17 03:17:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:32.503Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_processes\", asset=\"twa-tsgx05\", asset_id=\"99\", container=\"metrics-server\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545768544
Jul 17 03:17:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:32.515Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_resident_memory_bytes\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-udevd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689545787691
Jul 17 03:17:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:32.521Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_observed_generation\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"ipmi-exporter-6f8464dd94\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545784068
Jul 17 03:17:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:32.552Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-infiniband-umad7.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689545787691
Jul 17 03:17:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:32.562Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_bytes_total\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"vethc06831a0\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545780939
Jul 17 03:17:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:32.568Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_errs_total\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth2cff15d8\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545780939
Jul 17 03:17:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:32.590Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_annotations\", asset=\"msh-tsgx11\", asset_id=\"111\", configmap=\"kube-root-ca.crt\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-public\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545779276
Jul 17 03:17:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:32.591Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"124\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545780939
Jul 17 03:17:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:32.607Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"39\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545780791
Jul 17 03:17:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:32.657Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_terminated\", asset=\"twa-tsgx05\", asset_id=\"99\", container=\"prometheus-alertmanager\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-alertmanager-6df7f457b6-49ggg\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"32b13371-dbfd-40eb-bd71-e35dd195f135\"}" timestamp=1689545779894
Jul 17 03:17:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:32.708Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_last_seen\", asset=\"msh-tsgx05\", asset_id=\"67\", container=\"ipmi-exporter\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545775867
Jul 17 03:17:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:32.730Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"dign-server-p4cvw\", project=\"TSG-OS\", project_id=\"7\", reason=\"Shutdown\", service=\"prometheus-kube-state-metrics\", uid=\"b3068a5c-f8e2-4c2d-89be-7648e87ab2c1\"}" timestamp=1689545790436
Jul 17 03:17:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:32.745Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_labels\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.tsg-traffic-engine-numa-1.v1\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545790436
Jul 17 03:17:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:32.747Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failcnt\", asset=\"msh-tsgx05\", asset_id=\"67\", container=\"prometheus-node-exporter\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545763403
Jul 17 03:17:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:32.752Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"115\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545780939
Jul 17 03:17:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:32.763Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"twa-tsgx05\", asset_id=\"99\", container=\"exporter-merger\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"uninterruptible\"}" timestamp=1689545768940
Jul 17 03:17:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:32.768Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"microcode.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689545781015
Jul 17 03:17:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:32.769Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_address_not_ready\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_endpoint=\"traefik\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545784068
Jul 17 03:17:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:32.787Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_status_completion_time\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", job_name=\"helm-install-tsg-traffic-engine-numa-1\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545784068
Jul 17 03:17:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:32.796Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"67\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545779093
Jul 17 03:17:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:32.853Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"49\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545780939
Jul 17 03:17:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:32.866Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"8\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545780939
Jul 17 03:17:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:32.874Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"tmpfs\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/var/lib/kubelet/pods/03082888-996f-401e-985d-3fa51721b5b3/volumes/kubernetes.io~projected/kube-api-access-9nstt\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545779093
Jul 17 03:17:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:32.944Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"113\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545779093
Jul 17 03:17:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:32.957Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", fstype=\"ext4\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/var/lib/kubelet/pods/1df6bd2b-0fea-4549-94c2-ddadc3bbc3e4/volume-subpaths/telegraf/telegraf/0\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545779093
Jul 17 03:17:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:32.986Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_errors_received_total\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_0\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545779093
Jul 17 03:17:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:33.054Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"18\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545779093
Jul 17 03:17:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:33.127Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_io_threads_active\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545786438
Jul 17 03:17:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:33.196Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"twa-tsgx05\", asset_id=\"99\", container=\"metrics-server\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"sleeping\"}" timestamp=1689545768544
Jul 17 03:17:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:33.254Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", phase=\"Failed\", pod=\"dign-client-s49j9\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"2096ff4a-92d2-47a7-9b70-293a6568cee4\"}" timestamp=1689545779276
Jul 17 03:17:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:33.266Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx05\", asset_id=\"99\", container=\"dign-server-dns\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Total\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545768922
Jul 17 03:17:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:33.280Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"10\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545780605
Jul 17 03:17:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:33.335Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db209\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545776711
Jul 17 03:17:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:33.340Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", phase=\"Running\", pod=\"dign-server-xqfxk\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"e2593736-e404-42cd-a92c-c9652757c95a\"}" timestamp=1689545779276
Jul 17 03:17:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:33.352Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_commands_duration_seconds_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cmd=\"zadd\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545776711
Jul 17 03:17:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:33.379Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-a6a6d55f707d1c68a23401baa5eb13421890c13b98f06929bc285c140407ae60-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689545782644
Jul 17 03:17:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:33.559Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_mtu_bytes\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethf1fa10d9\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545780605
Jul 17 03:17:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:33.576Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_mod_info\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"github.com/prometheus/client_golang\", nz_agent_id=\"3\", program=\"systemd_exporter\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", version=\"v1.5.1\"}" timestamp=1689545787691
Jul 17 03:17:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:33.729Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"31\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545780605
Jul 17 03:17:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:33.742Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"68\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545780605
Jul 17 03:17:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:33.761Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_created\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"metrics-server\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545779894
Jul 17 03:17:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:33.765Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"twa-tsgx05\", asset_id=\"99\", condition=\"unknown\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"cm-policy-local-cache-redis-master-69967b5879-vlgl8\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"e8ef10b6-3745-4720-b3f4-f80080df3d45\"}" timestamp=1689545779894
Jul 17 03:17:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:33.782Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"51\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545780605
Jul 17 03:17:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:33.829Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_labels\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_endpoint=\"prometheus-alertmanager\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545779894
Jul 17 03:17:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:33.940Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545780605
Jul 17 03:17:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:33.962Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_name_assign_type\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth45bb7033\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545780605
Jul 17 03:17:38 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:38.164Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kafka_network_RequestMetrics_Errors_total\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-Kafka-twa-dellr740-dt001\", endpoint_id=\"105\", error=\"NONE\", module=\"DC-Kafka\", module_id=\"5\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", request=\"DescribeGroups\"}" timestamp=1689545797482
Jul 17 03:17:39 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:39.621Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_scaling_frequency_min_hertz\", asset=\"twa-dellr740-dt003\", asset_id=\"41\", cpu=\"31\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"OLAP-Node-Exporter-twa-dellr740-dt003\", endpoint_id=\"261\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"hbase_master,clickhouse_data,kafka,gohangout,zookeeper,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545798473
Jul 17 03:17:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:40.536Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_carrier_total\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth3d4cfc49\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545791647
Jul 17 03:17:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:40.565Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"116\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545791647
Jul 17 03:17:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:41.069Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db78\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545793827
Jul 17 03:17:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:41.078Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_protocol_type\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens2f1np1\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545791647
Jul 17 03:17:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:41.119Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_annotations\", asset=\"twa-tsgx04\", asset_id=\"97\", configmap=\"telegraf-tsg-traffic-engine-numa-0\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545800034
Jul 17 03:17:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:41.353Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_protocol_type\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens4f2np2\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545791647
Jul 17 03:17:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:41.363Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"overlay\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-a62b48d6dd1421c6867004f39c2ec975b036ee3ece510ba20c3b05166b9b9fce-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689545795764
Jul 17 03:17:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:41.372Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_scrape_collector_success\", asset=\"twa-tsgx04\", asset_id=\"97\", collector=\"xfs\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545791647
Jul 17 03:17:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:41.427Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_errs_total\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethb6572f7f\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545791647
Jul 17 03:17:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:41.479Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_dormant\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethf0bd7ef8\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545791647
Jul 17 03:17:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:41.497Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_packets_total\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens8f1np1\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545791647
Jul 17 03:17:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:41.536Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_load_average_10s\", asset=\"twa-tsgx04\", asset_id=\"97\", container=\"local-path-provisioner\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545773090
Jul 17 03:17:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:41.541Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_filteredReadRequestCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", namespace=\"SYSTEM\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"495b2450babc8ed5d1ad40d78a20bebb\", table=\"LOG\", type=\"data\"}" timestamp=1689545799757
Jul 17 03:17:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:41.552Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_labels\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"coredns\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545800034
Jul 17 03:17:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:41.559Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_cache\", asset=\"twa-tsgx04\", asset_id=\"97\", container=\"prometheus-node-exporter\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545782680
Jul 17 03:17:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:41.573Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas_updated\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"systemd-exporter\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545800034
Jul 17 03:17:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:41.674Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_waiting\", asset=\"twa-tsgx04\", asset_id=\"97\", container=\"telegraf-security\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", pod=\"tsg-traffic-engine-numa-1-firewall-7bff8557d6-cnnft\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"9ad11045-e0bd-4481-9c2d-4c81a67475ee\"}" timestamp=1689545800034
Jul 17 03:17:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:41.857Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_10_193_1_metric_splitTime_99_9th_percentile\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545799757
Jul 17 03:17:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:41.881Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_10_193_1_metric_flushOutputSize_mean\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545799757
Jul 17 03:17:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:42.054Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_putCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"session_record_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"6543c0c81087304ffabcd0d3fe61e06b\", table=\"index_filename_session_record_hos_bucket\", type=\"index\"}" timestamp=1689545799757
Jul 17 03:17:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:42.072Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_appendCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"troubleshooting_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"f942b1b1098244a4ec1b2578b462bd41\", table=\"index_time_troubleshooting_hos_bucket\", type=\"index\"}" timestamp=1689545799757
Jul 17 03:17:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:42.094Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_10_194_1_metric_putBatchTime_mean\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545799757
Jul 17 03:17:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:42.115Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_minStoreFileAge\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"93a02ca4d08e8233247927e2ad0aa8d1\", table=\"session_record_hos_bucket\", type=\"data\"}" timestamp=1689545799757
Jul 17 03:17:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:42.171Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_threads\", asset=\"twa-tsgx04\", asset_id=\"97\", container=\"dign-server-dns\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545780769
Jul 17 03:17:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:42.212Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"twa-tsgx04\", asset_id=\"97\", container=\"exporter\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"sleeping\"}" timestamp=1689545784561
Jul 17 03:17:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:42.232Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_labels\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-alertmanager\", uid=\"36dcf15d-dc40-4d9f-b620-a0ea68088357\"}" timestamp=1689545800034
Jul 17 03:17:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:42.268Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_replicaid\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"default\", project=\"OLAP\", project_id=\"3\", region=\"4fc005329c2eab495e1dfadd3fe1163f\", table=\"index_partfile_default\", type=\"index\"}" timestamp=1689545799757
Jul 17 03:17:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:42.411Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"scrape_samples_scraped\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_instance=\"10.10.10.161:9100\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545796242
Jul 17 03:17:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:42.489Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_discards_transmitted_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"mlx5_8\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545796242
Jul 17 03:17:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:42.509Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-sda3.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689545800671
Jul 17 03:17:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:42.512Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_cpu_shares\", asset=\"pcap-tsgx01\", asset_id=\"71\", container=\"dign-server-web\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545800951
Jul 17 03:17:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:42.535Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"java_lang_MemoryPool_UsageThresholdCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"Metaspace\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545799757
Jul 17 03:17:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:42.551Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_system_table_bucket_meta_metric_splitTime_mean\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545799757
Jul 17 03:17:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:42.571Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_goroutines\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545794838
Jul 17 03:17:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:42.623Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_deleteCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", namespace=\"SYSTEM\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"e1edba3b81b21a74ea026f21ba7bed91\", table=\"LOG\", type=\"data\"}" timestamp=1689545799757
Jul 17 03:17:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:42.642Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_hbase_table_meta_metric_appendTime_95th_percentile\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545799757
Jul 17 03:17:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:42.643Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_numFilesCompactedCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"a61e2634ced65a63813ac972443dc15e\", table=\"assessment_hos_bucket\", type=\"data\"}" timestamp=1689545799757
Jul 17 03:17:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:42.740Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_system_table_bucket_meta_metric_splitTime_90th_percentile\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545799757
Jul 17 03:17:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:42.886Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"getty.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"target\"}" timestamp=1689545800671
Jul 17 03:17:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:42.952Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_virtual_memory_max_bytes\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"rshim.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689545800671
Jul 17 03:17:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:43.000Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", cpu=\"35\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545796242
Jul 17 03:17:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:43.172Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_compressed_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens4f1np1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545796242
Jul 17 03:17:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:43.235Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_spec_strategy_rollingupdate_max_unavailable\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"prometheus-server\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545795154
Jul 17 03:17:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:43.254Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"abrtd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689545800671
Jul 17 03:17:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:43.271Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", cpu=\"7\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545796242
Jul 17 03:17:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:43.384Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", container=\"firewall\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", state=\"sleeping\"}" timestamp=1689545797223
Jul 17 03:17:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:43.503Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_reads_bytes_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", container=\"metrics\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545791476
Jul 17 03:17:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:43.649Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", container=\"kubernetes-dashboard\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"iowaiting\"}" timestamp=1689545790466
Jul 17 03:17:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:43.749Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_labels\", asset=\"pcap-tsgx01\", asset_id=\"71\", configmap=\"kube-root-ca.crt\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-public\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545795154
Jul 17 03:17:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:43.757Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_writes_bytes_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", container=\"kubernetes-dashboard\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545790466
Jul 17 03:17:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:43.775Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_bytes_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"cadvisor\", interface=\"eno2\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545796308
Jul 17 03:17:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:43.790Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"pcap-tsgx01\", asset_id=\"71\", condition=\"Available\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"prometheus-server\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"unknown\"}" timestamp=1689545795154
Jul 17 03:17:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:43.815Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db135\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545794838
Jul 17 03:17:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:43.823Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_heap_objects\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Gohangout-msh-dellr740-dt002\", endpoint_id=\"167\", module=\"DC-Gohangout\", module_id=\"35\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545802623
Jul 17 03:17:47 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:47.000Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"java_lang_Runtime_StartTime\", asset=\"pcap-dellr740-dt003\", asset_id=\"35\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt003-Hmaster\", endpoint_id=\"387\", module=\"DC-HBase\", module_id=\"3\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545806370
Jul 17 03:17:48 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:48.705Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_numBytesCompactedCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"e290d75c573af3244c710e999f537395\", table=\"proxy_hos_bucket\", type=\"data\"}" timestamp=1689545807439
Jul 17 03:17:48 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:48.775Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_appendCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"9d091b0c3b3083399f0cc4d765f30b7d\", table=\"session_record_hos_bucket\", type=\"data\"}" timestamp=1689545807439
Jul 17 03:17:48 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:48.792Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_2_metric_flushMemstoreSize_99_9th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545807439
Jul 17 03:17:48 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:48.826Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_tsg_galaxy_table_relation_account_framedip_metric_tableReadQueryPerSecond_15min_rate\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545807439
Jul 17 03:17:48 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:48.880Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kafka_consumergroup_lag\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", consumergroup=\"relationship-gtpc-user-20220830-1\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"Kafka-Lag-msh-dellr740-dt001\", endpoint_id=\"263\", module=\"Kafka-Lag\", module_id=\"25\", nz_agent_id=\"3\", partition=\"0\", project=\"OLAP\", project_id=\"3\", topic=\"GTPC-RECORD-COMPLETED\"}" timestamp=1689545808567
Jul 17 03:17:48 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:48.977Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_LOG_metric_appendTime_min\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545807439
Jul 17 03:17:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:49.028Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_193_1_metric_scanTime_98th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545807439
Jul 17 03:17:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:49.041Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_193_1_metric_appendTime_min\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545807439
Jul 17 03:17:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:49.053Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_dos_table_ddos_traffic_baselines_metric_putTime_99th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545807439
Jul 17 03:17:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:49.065Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_IndexPrepareTime_median\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"PhoenixIndexer\"}" timestamp=1689545807439
Jul 17 03:17:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:49.146Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_hbase_table_namespace_metric_compactionInputSize_max\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545807439
Jul 17 03:17:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:49.162Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_session_record_hos_bucket_metric_majorCompactionOutputFileCount_98th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545807439
Jul 17 03:17:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:49.177Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_tsg_galaxy_table_gtpc_knowledge_base_metric_tableReadQueryPerSecond_15min_rate\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545807439
Jul 17 03:17:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:49.178Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_readRequestCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"bc887690f0e0fbc3952eeae971f001a4\", table=\"troubleshooting_hos_bucket\", type=\"data\"}" timestamp=1689545807439
Jul 17 03:17:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:49.194Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_numBytesCompactedCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"firewall_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"b7d86e59d00af81edd1d1c354d8c0f1a\", table=\"index_time_firewall_hos_bucket\", type=\"index\"}" timestamp=1689545807439
Jul 17 03:17:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:49.241Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_session_record_hos_bucket_metric_flushOutputSize_max\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545807439
Jul 17 03:17:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:49.262Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_session_record_hos_bucket_metric_compactionInputFileCount_99_9th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545807439
Jul 17 03:17:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:49.274Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_193_1_metric_compactionOutputFileCount_min\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545807439
Jul 17 03:17:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:49.404Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_filteredReadRequestCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"troubleshooting_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"b70eb299f64649891465978a22b50d75\", table=\"index_partfile_troubleshooting_hos_bucket\", type=\"index\"}" timestamp=1689545807439
Jul 17 03:17:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:49.438Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"java_lang_MemoryPool_UsageThresholdSupported\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"Par Eden Space\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545807439
Jul 17 03:17:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:49.477Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_maxFlushQueueSize\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"rtp_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"8b4bb800fda3b83e73863db9941293cf\", table=\"index_partfile_rtp_hos_bucket\", type=\"index\"}" timestamp=1689545807439
Jul 17 03:17:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:49.531Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_compactionsCompletedCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"rtp_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"b242966859aa9721f3ac6e8b17d51a4d\", table=\"index_filename_rtp_hos_bucket\", type=\"index\"}" timestamp=1689545807439
Jul 17 03:17:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:49.610Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_194_3_metric_flushTime_99th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545807439
Jul 17 03:17:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:49.641Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_filteredReadRequestCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"81707482c620eb510b89508d73a4c9f7\", table=\"knowledge_base_hos_bucket\", type=\"data\"}" timestamp=1689545807439
Jul 17 03:17:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:49.667Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_4_metric_flushOutputSize_99_9th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545807439
Jul 17 03:17:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:49.670Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_readRequestCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"default\", project=\"OLAP\", project_id=\"3\", region=\"1ec6c491dc19e54814be0f747b7ceb69\", table=\"index_time_default\", type=\"index\"}" timestamp=1689545807439
Jul 17 03:17:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:49.689Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_blockCacheRootIndexMissCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Server\"}" timestamp=1689545807439
Jul 17 03:17:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:49.773Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_bytes_total\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens8f0np0\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545801797
Jul 17 03:17:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:49.795Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-8bb79bea\\\\x2d895f\\\\x2d4fbf\\\\x2dbba7\\\\x2dff346d91b1a0-volume\\\\x2dsubpaths-config\\\\x2dvolume-telegraf\\\\x2dsecurity-2.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689545801057
Jul 17 03:17:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:49.840Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_complete\", asset=\"twa-tsgx10\", asset_id=\"109\", condition=\"false\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", job_name=\"helm-install-systemd-exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545799668
Jul 17 03:17:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:49.898Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_4_metric_scanSize_99_9th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545807439
Jul 17 03:17:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:49.916Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_LOG_metric_scanTime_90th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545807439
Jul 17 03:17:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:49.940Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_3_metric_tableReadQueryPerSecond_mean_rate\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545807439
Jul 17 03:17:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:50.051Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_compressed_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth66a8f63b\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545806709
Jul 17 03:17:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:50.089Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_reads_merged_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"overlay_0-78\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545797972
Jul 17 03:17:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:50.183Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"44\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545806709
Jul 17 03:17:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:50.197Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:80-0000:80:01.1-0000:81:00.0-host0-target0:2:0-0:2:0:0-block-sda-sda2.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689545806554
Jul 17 03:17:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:50.347Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"40\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545806709
Jul 17 03:17:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:50.407Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", cpu=\"17\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-olap003\", endpoint_id=\"235\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"yarn_resourcemanager,zookeeper,nacos,kafka,spark,druid_data,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545809713
Jul 17 03:17:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:50.441Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx08\", asset_id=\"105\", container=\"kubernetes-dashboard\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Sync\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545801112
Jul 17 03:17:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:50.469Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_metadata_resource_version\", asset=\"twa-tsgx08\", asset_id=\"105\", configmap=\"kube-root-ca.crt\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-public\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545801568
Jul 17 03:17:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:50.565Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx10\", asset_id=\"109\", cpu=\"4\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545801797
Jul 17 03:17:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:50.633Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_info\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_endpoint=\"prometheus-alertmanager\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545801568
Jul 17 03:17:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:50.663Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_frequency_min_hertz\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"9\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545806709
Jul 17 03:17:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:50.666Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_labels\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-traefik-nw8gw\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"10837ce3-9967-4da8-9ad8-57d8794e08a8\"}" timestamp=1689545801568
Jul 17 03:17:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:50.693Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-pod7b43283a_781d_45c1_8760_e8b3248f8fd2.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"slice\"}" timestamp=1689545801057
Jul 17 03:17:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:50.714Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_errors_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", interface=\"bond0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545806617
Jul 17 03:17:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:50.717Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:00-0000:00:01.1-0000:01:00.0-infiniband-mlx5_0.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689545801057
Jul 17 03:17:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:50.734Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"phy_rx_error_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", dev=\"virtio_dign_s\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"mrzcpd-exporter\"}" timestamp=1689545801366
Jul 17 03:17:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:50.784Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-systemd-exporter-ns8p4\", project=\"TSG-OS\", project_id=\"7\", reason=\"Shutdown\", service=\"prometheus-kube-state-metrics\", uid=\"132271d2-be10-4fb8-8ec9-3f73cc203cb2\"}" timestamp=1689545801568
Jul 17 03:17:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:50.822Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"17\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545806709
Jul 17 03:17:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:50.868Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", cpu=\"40\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-olap003\", endpoint_id=\"235\", mode=\"idle\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"yarn_resourcemanager,zookeeper,nacos,kafka,spark,druid_data,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545809713
Jul 17 03:17:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:51.053Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"auditd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689545806554
Jul 17 03:17:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:51.421Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_size_bytes\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"shm\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/3b940f5034a36f26e5912606cab9514359054bbcab4e60b30f5f852459772e99/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545801797
Jul 17 03:17:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:51.447Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_io_time_weighted_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/var/lib/kubelet/pods/32707ee1-6f45-4d2c-8e0e-78edc233adc3/volumes/kubernetes.io~projected/kube-api-access-x64jh\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545797972
Jul 17 03:17:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:51.531Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-mrzcpd-exporter-zjhth\", project=\"TSG-OS\", project_id=\"7\", reason=\"NodeAffinity\", service=\"prometheus-kube-state-metrics\", uid=\"a88ca435-a295-43d3-bffd-a3842ce86c5e\"}" timestamp=1689545801568
Jul 17 03:17:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:51.717Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"ext4\", name=\"var-lib-kubelet-pods-6f7082c9\\\\x2d24a1\\\\x2d4f4f\\\\x2daf11\\\\x2dba474eb9a99b-volume\\\\x2dsubpaths-config\\\\x2dvolume-telegraf\\\\x2dsecurity-2.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689545806554
Jul 17 03:17:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:51.838Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_last_seen\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689545807595
Jul 17 03:17:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:51.861Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_packets_dropped_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", interface=\"ens2f0np0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545806617
Jul 17 03:17:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:51.873Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_annotations\", asset=\"twa-tsgx08\", asset_id=\"105\", configmap=\"prometheus-server\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545801568
Jul 17 03:17:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:51.890Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_up_changes_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth863d4c07\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545806709
Jul 17 03:17:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:52.549Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_persistentvolumeclaim_status_phase\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", persistentvolumeclaim=\"prometheus-alertmanager\", phase=\"Bound\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545804478
Jul 17 03:17:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:52.659Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"auth-rpcgss-module.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689545805496
Jul 17 03:17:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:52.697Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", container=\"telegraf-security\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", state=\"iowaiting\"}" timestamp=1689545801449
Jul 17 03:17:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:53.189Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db241\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545799367
Jul 17 03:17:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:53.213Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_daemonset_status_current_number_scheduled\", asset=\"msh-tsgx08\", asset_id=\"65\", daemonset=\"svclb-traefik-ab31cfb4\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545808950
Jul 17 03:17:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:53.247Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_info\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", job_name=\"helm-install-systemd-exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545808950
Jul 17 03:17:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:53.260Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"msh-tsgx12\", asset_id=\"113\", cpu=\"10\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545803391
Jul 17 03:17:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:53.403Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_terminated_reason\", asset=\"msh-tsgx08\", asset_id=\"65\", container=\"helm\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-tsg-traffic-engine-numa-1-glkhp\", project=\"TSG-OS\", project_id=\"7\", reason=\"Completed\", service=\"prometheus-kube-state-metrics\", uid=\"8ec229fc-9306-4096-93ee-079996a02e4e\"}" timestamp=1689545808950
Jul 17 03:17:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:53.617Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"vethb1d236b8\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545808328
Jul 17 03:17:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:53.655Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"msh-tsgx08\", asset_id=\"65\", condition=\"Progressing\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"mrzcpd-exporter\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"true\"}" timestamp=1689545808950
Jul 17 03:17:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:53.696Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx12\", asset_id=\"113\", cpu=\"86\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545803391
Jul 17 03:17:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:53.718Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_mtu_bytes\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"br_dign_s\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545803391
Jul 17 03:17:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:53.743Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"60\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545809337
Jul 17 03:17:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:53.851Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", container=\"metrics-server\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Read\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545805284
Jul 17 03:17:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:53.882Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled_time\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-node-exporter-twdv4\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"532996fd-c16e-43f2-a8fc-62dbb64b4c49\"}" timestamp=1689545802058
Jul 17 03:17:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:53.894Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_transmit_wait_total\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_7\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545808328
Jul 17 03:17:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:53.962Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_processes\", asset=\"pcap-tsgx07\", asset_id=\"85\", container=\"prometheus-server-configmap-reload\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545799399
Jul 17 03:17:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:54.092Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cpu=\"116\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545799409
Jul 17 03:17:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:54.123Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_hwmon_chip_names\", asset=\"pcap-tsgx07\", asset_id=\"85\", chip=\"0000:80:01_2_0000:82:00_0\", chip_name=\"bnxt_en\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545799409
Jul 17 03:17:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:54.183Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_spec_replicas\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"prometheus-kube-state-metrics-6b768c7bf8\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545802058
Jul 17 03:17:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:54.185Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cpu=\"64\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545799409
Jul 17 03:17:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:54.211Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_local_link_integrity_errors_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"mlx5_8\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545799409
Jul 17 03:17:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:54.213Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cpu=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545799409
Jul 17 03:17:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:54.376Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_errs_total\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"br_dign_s\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545803391
Jul 17 03:17:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:54.385Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_changes_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth7f2a971d\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545799409
Jul 17 03:17:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:54.409Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_mapped_file\", asset=\"msh-tsgx08\", asset_id=\"65\", container=\"metrics\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545794700
Jul 17 03:17:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:54.431Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_metadata_generation\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"tsg-traffic-engine-numa-0-firewall\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545808950
Jul 17 03:17:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:54.522Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_created\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", pod=\"tsg-traffic-engine-numa-1-firewall-6c94f9df78-jgzzt\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"bdb49414-5fd6-4eeb-a829-ad76bdcb350b\"}" timestamp=1689545802058
Jul 17 03:17:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:54.590Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx08\", asset_id=\"65\", cpu=\"4\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545808328
Jul 17 03:17:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:54.638Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_commands_rejected_calls_total\", asset=\"msh-tsgx08\", asset_id=\"65\", cmd=\"multi\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545810108
Jul 17 03:17:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:54.708Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_multicast_total\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens8f0np0\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545803391
Jul 17 03:17:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:54.726Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_packets_total\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth149fb130\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545803391
Jul 17 03:17:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:54.941Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cpu=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545799409
Jul 17 03:17:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:54.942Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_namespace_annotations\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545808663
Jul 17 03:17:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:54.982Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx12\", asset_id=\"113\", cpu=\"100\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545803391
Jul 17 03:17:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:54.985Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_usage_bytes\", asset=\"pcap-tsgx07\", asset_id=\"85\", container=\"prometheus-alertmanager-configmap-reload\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545790474
Jul 17 03:17:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:55.005Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_udp_queues\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", ip=\"v6\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", queue=\"rx\", service=\"prometheus-node-exporter\"}" timestamp=1689545799409
Jul 17 03:17:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:55.007Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_max_fds\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"udisks2.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689545812135
Jul 17 03:17:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:55.009Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"msh-tsgx08\", asset_id=\"65\", cpu=\"112\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545808328
Jul 17 03:17:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:55.119Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx12\", asset_id=\"113\", cpu=\"35\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545803391
Jul 17 03:17:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:55.119Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_writes_total\", asset=\"msh-tsgx12\", asset_id=\"113\", container=\"prometheus-pushgateway\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545797977
Jul 17 03:17:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:55.134Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"msh-tsgx12\", asset_id=\"113\", container=\"mrapm-device\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"uninterruptible\"}" timestamp=1689545787133
Jul 17 03:17:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:55.134Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx12\", asset_id=\"113\", cpu=\"74\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545803391
Jul 17 03:17:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:55.157Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_sockstat_TCP_tw\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545803391
Jul 17 03:17:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:55.222Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"pcap-tsgx07\", asset_id=\"85\", condition=\"unknown\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"traefik-64f55bb67d-bcw8n\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"b9686156-a602-47ce-b798-d231f10c8f92\"}" timestamp=1689545802058
Jul 17 03:17:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:55.316Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_tasks_max\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"polkit.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689545800884
Jul 17 03:17:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:55.464Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_physical_state_id\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"mlx5_3\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545799409
Jul 17 03:17:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:55.494Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_packets_transmitted_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"mlx5_7\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545799409
Jul 17 03:17:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:55.677Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"promhttp_metric_handler_requests_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", code=\"503\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545799409
Jul 17 03:17:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:56.357Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", container=\"prometheus-node-exporter\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"uninterruptible\"}" timestamp=1689545808707
Jul 17 03:17:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:56.594Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"109\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545805883
Jul 17 03:17:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:56.727Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_bytes_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"cadvisor\", interface=\"eno34np1\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545808668
Jul 17 03:17:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:56.759Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"16\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545805883
Jul 17 03:17:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:56.791Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"119\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545805883
Jul 17 03:17:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:56.874Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", container=\"dashboard-metrics-scraper\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"cadvisor\", failure_type=\"pgmajfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"container\"}" timestamp=1689545795439
Jul 17 03:17:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:56.918Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"108\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545805883
Jul 17 03:17:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:56.921Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_max_fds\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-logind.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689545803372
Jul 17 03:17:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:56.981Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"iscsi-shutdown.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"service\"}" timestamp=1689545803372
Jul 17 03:17:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:56.990Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", container=\"configmap-reloader\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Discard\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545810415
Jul 17 03:17:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:56.990Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_observed_generation\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"kubernetes-dashboard-66dd5b456\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545813389
Jul 17 03:17:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:56.991Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_fifo_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"vethdfe7d9ac\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545805883
Jul 17 03:17:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:57.007Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"106\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545805883
Jul 17 03:17:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:57.009Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_processes\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689545808321
Jul 17 03:17:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:57.041Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_scrape_collector_success\", asset=\"pcap-tsgx03\", asset_id=\"73\", collector=\"diskstats\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545805883
Jul 17 03:17:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:57.043Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_start_time\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-ipmi-exporter-5v575\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"c861b65c-0434-420a-b012-9cf2a1d7ad39\"}" timestamp=1689545813389
Jul 17 03:17:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:57.058Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_colls_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens4f1np1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545805883
Jul 17 03:17:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:57.080Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_packets_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"cadvisor\", interface=\"ens6f1np1\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545808668
Jul 17 03:17:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:57.100Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_restarts_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", container=\"helm\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-ipmi-exporter-5v575\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"c861b65c-0434-420a-b012-9cf2a1d7ad39\"}" timestamp=1689545813389
Jul 17 03:17:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:57.125Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_nf_conntrack_stat_early_drop\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545805883
Jul 17 03:17:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:57.125Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_data_received_bytes_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"mlx5_5\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545805883
Jul 17 03:17:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:57.183Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-40f27e3b\\\\x2df7e6\\\\x2d47e8\\\\x2da251\\\\x2dded132fab39b-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2d2zg5m.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689545803372
Jul 17 03:17:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:57.265Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-66bcc6c3671b9053123ce02a1f64e69ed7d571a39ce3de8ad4d5e16ab37d4ee7.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"scope\"}" timestamp=1689545803372
Jul 17 03:17:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:57.296Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-netns-cni\\\\x2d95f8c411\\\\x2d21ab\\\\x2d04e8\\\\x2d62b4\\\\x2da1404fe3f860.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689545803372
Jul 17 03:17:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:57.332Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"88\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545805883
Jul 17 03:17:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:57.820Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_data_received_bytes_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"mlx5_8\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545805883
Jul 17 03:17:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:57.839Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_address_assign_type\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth958fa935\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545805883
Jul 17 03:17:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:57.939Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545805883
Jul 17 03:17:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:57.962Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"42\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545805883
Jul 17 03:17:58 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:58.249Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"106\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545805883
Jul 17 03:17:58 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:58.293Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_carrier_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"eno1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545805883
Jul 17 03:17:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:59.721Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"msh-tsgx01\", asset_id=\"51\", container=\"mrapm-device\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"iowaiting\"}" timestamp=1689545797663
Jul 17 03:17:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:59.733Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_name_assign_type\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth844980e5\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545809083
Jul 17 03:17:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:59.743Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_hwmon_temp_celsius\", asset=\"msh-tsgx01\", asset_id=\"51\", chip=\"pci0000:00_0000:00:18_3\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", sensor=\"temp4\", service=\"prometheus-node-exporter\"}" timestamp=1689545809083
Jul 17 03:17:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:17:59.957Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_info\", address=\"1a:af:56:3e:d7:b2\", asset=\"msh-tsgx01\", asset_id=\"51\", broadcast=\"ff:ff:ff:ff:ff:ff\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"vethb97771d9\", duplex=\"full\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operstate=\"up\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545809083
Jul 17 03:18:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:00.733Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"1\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545809083
Jul 17 03:18:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:00.792Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_id\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"eno36np3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545809083
Jul 17 03:18:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:00.856Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_timex_tick_seconds\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545809083
Jul 17 03:18:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:00.881Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"snmpd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"notify\", type=\"service\"}" timestamp=1689545804879
Jul 17 03:18:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:00.944Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_metadata_generation\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"systemd-exporter-7f549d9485\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545807425
Jul 17 03:18:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:00.966Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_mtu_bytes\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"eno2\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545809083
Jul 17 03:18:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:01.148Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"77\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545809083
Jul 17 03:18:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:01.153Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_init_container_status_terminated\", asset=\"msh-tsgx01\", asset_id=\"51\", container=\"firewall-init\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", pod=\"tsg-traffic-engine-numa-1-firewall-6c6f4bc877-xxxz9\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"b3efc35a-4844-47d9-82c9-7d335d0d4eaa\"}" timestamp=1689545807425
Jul 17 03:18:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:01.176Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_memory_used_scripts_bytes\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545805071
Jul 17 03:18:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:01.221Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"overlay\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-e856c80aa42fed93b666eabade8e204351a2fdb09db6910acf7e073c12c21a12-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689545804879
Jul 17 03:18:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:01.233Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-mrzcpd-exporter-6xctw\", project=\"TSG-OS\", project_id=\"7\", reason=\"UnexpectedAdmissionError\", service=\"prometheus-kube-state-metrics\", uid=\"64e78458-5407-4009-8438-11206cb5f43d\"}" timestamp=1689545807425
Jul 17 03:18:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:01.266Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"60\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545809083
Jul 17 03:18:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:01.408Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_metadata_generation\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"ipmi-exporter-6f8464dd94\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545807425
Jul 17 03:18:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:01.467Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"77\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545809083
Jul 17 03:18:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:01.492Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_data_transmitted_bytes_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_2\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545809083
Jul 17 03:18:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:01.574Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_scrape_collector_duration_seconds\", asset=\"msh-tsgx01\", asset_id=\"51\", collector=\"xfs\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545809083
Jul 17 03:18:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:02.679Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_owner\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", job_name=\"helm-install-cm-policy-local-cache\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", owner_is_controller=\"<none>\", owner_kind=\"<none>\", owner_name=\"<none>\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545811412
Jul 17 03:18:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:02.745Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"27\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545818318
Jul 17 03:18:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:03.359Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"nss-lookup.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"target\"}" timestamp=1689545808447
Jul 17 03:18:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:03.387Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"net_conntrack_dialer_conn_failed_total\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", dialer_name=\"61\", instance=\"10.10.20.159:10090\", reason=\"timeout\", type=\"global\"}" timestamp=1689545822989
Jul 17 03:18:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:03.398Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-d8eb5322c7ab186db87d1e0d2199818b9c23dd38b3bea3d3f60c5540748040c7-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689545808447
Jul 17 03:18:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:03.453Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx02\", asset_id=\"63\", container=\"dign-client\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Sync\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545795917
Jul 17 03:18:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:03.485Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_labels\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", job_name=\"helm-install-traefik-crd\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545811412
Jul 17 03:18:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:03.507Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-delete-tsg-traffic-engine-numa-1-ddm55\", project=\"TSG-OS\", project_id=\"7\", reason=\"Evicted\", service=\"prometheus-kube-state-metrics\", uid=\"de0dc5f3-860f-415f-8f70-95817cf0299d\"}" timestamp=1689545811412
Jul 17 03:18:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:03.566Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_bytes_total\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth973158a9\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545818318
Jul 17 03:18:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:03.569Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"msh-tsgx02\", asset_id=\"63\", cpu=\"78\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545818318
Jul 17 03:18:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:03.595Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"prometheus_target_scrape_pool_sync_total\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", instance=\"10.10.20.159:10090\", scrape_job=\"asset_ping_5\", type=\"global\"}" timestamp=1689545822989
Jul 17 03:18:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:03.664Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_fifo_total\", asset=\"pcap-dellr740-dt002\", asset_id=\"33\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"lo\", endpoint=\"OLAP-Node-Exporter-pcap-dellr740-dt002\", endpoint_id=\"253\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545822497
Jul 17 03:18:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:03.745Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"msh-tsgx02\", asset_id=\"63\", container=\"prometheus-server\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", failure_type=\"pgfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"hierarchy\"}" timestamp=1689545804808
Jul 17 03:18:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:03.808Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_rapl_package_joules_total\", asset=\"msh-dellr740-olap005\", asset_id=\"19\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Spark-msh-dellr740-olap005\", endpoint_id=\"17\", index=\"1\", module=\"NC-Spark\", module_id=\"47\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545823391
Jul 17 03:18:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:03.916Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_state_started\", asset=\"msh-tsgx02\", asset_id=\"63\", container=\"exporter-merger\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"mrzcpd-exporter-5cf7f967cc-9zrnz\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"0b1be0e3-e71c-4f3b-83a7-1d29910105b8\"}" timestamp=1689545811412
Jul 17 03:18:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:03.928Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"net_conntrack_dialer_conn_established_total\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", dialer_name=\"67\", instance=\"10.10.20.159:10090\", type=\"global\"}" timestamp=1689545822989
Jul 17 03:18:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:03.966Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"prometheus_target_scrape_pool_targets\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", instance=\"10.10.20.159:10090\", scrape_job=\"483\", type=\"global\"}" timestamp=1689545822989
Jul 17 03:18:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:04.006Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx02\", asset_id=\"63\", container=\"mrapm-device\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Discard\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545802769
Jul 17 03:18:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:04.060Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_packets_transmitted_total\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_5\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545818318
Jul 17 03:18:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:04.074Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_errors_total\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", interface=\"ens4f3np3\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545811873
Jul 17 03:18:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:04.123Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_state_started\", asset=\"msh-tsgx02\", asset_id=\"63\", container=\"metrics-server\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"metrics-server-54dc485875-5f8l6\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"f27721d6-5ab0-4b36-a5e6-4963b6828dca\"}" timestamp=1689545811412
Jul 17 03:18:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:04.381Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_nf_conntrack_stat_ignore\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-dt001\", endpoint_id=\"225\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545822987
Jul 17 03:18:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:04.450Z caller=dedupe.go:112 component=remote level=info remote_name=52db6f url=http://10.10.10.159:10090/nz-agent/prometheus/proxy/api/v1/write msg="Dropped sample for series that was not explicitly dropped via relabelling" ref=4436650
Jul 17 03:18:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:04.583Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"net_conntrack_dialer_conn_failed_total\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", dialer_name=\"33\", instance=\"10.10.20.159:10090\", reason=\"refused\", type=\"global\"}" timestamp=1689545822989
Jul 17 03:18:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:04.893Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", cpu=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-dt001\", endpoint_id=\"225\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545822987
Jul 17 03:18:05 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:05.085Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", cpu=\"11\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-dt001\", endpoint_id=\"225\", mode=\"softirq\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545822987
Jul 17 03:18:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:06.666Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_limit_bytes\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda2\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545819206
Jul 17 03:18:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:06.958Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db138\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545818290
Jul 17 03:18:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:07.118Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", cpu=\"6\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Spark-msh-dellr740-olap003\", endpoint_id=\"13\", mode=\"steal\", module=\"NC-Spark\", module_id=\"47\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545826094
Jul 17 03:18:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:07.628Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_scaling_frequency_min_hertz\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", cpu=\"40\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Spark-msh-dellr740-olap003\", endpoint_id=\"13\", module=\"NC-Spark\", module_id=\"47\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545826094
Jul 17 03:18:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:07.685Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_io_current\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda3\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545819206
Jul 17 03:18:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:07.769Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cpu=\"2\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545823910
Jul 17 03:18:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:07.818Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_complete\", asset=\"twa-tsgx01\", asset_id=\"91\", condition=\"false\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", job_name=\"helm-install-ipmi-exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545821227
Jul 17 03:18:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:07.894Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_status_failed\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", job_name=\"helm-install-tsg-traffic-engine-numa-1\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545821227
Jul 17 03:18:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:07.911Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_replicas\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"systemd-exporter-7f549d9485\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545821227
Jul 17 03:18:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:07.988Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cpu=\"30\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545823910
Jul 17 03:18:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:08.341Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_spec_type\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", type=\"ClusterIP\", uid=\"13a04f91-0314-4890-8733-7d551f3d9bcc\"}" timestamp=1689545821227
Jul 17 03:18:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:08.378Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cpu=\"13\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545823910
Jul 17 03:18:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:08.386Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_last_seen\", asset=\"twa-tsgx01\", asset_id=\"91\", container=\"metrics\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545822985
Jul 17 03:18:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:08.422Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_cpu_seconds_total\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-poda99c67a0_fa8b_483f_afda_9724525367bb.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"slice\"}" timestamp=1689545813582
Jul 17 03:18:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:08.478Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db207\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545818290
Jul 17 03:18:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:08.512Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-burstable.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"slice\"}" timestamp=1689545813582
Jul 17 03:18:09 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:09.523Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_numBytesCompactedCount\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"troubleshooting_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"bf2a458b5ee504849c8f27ed46a87a35\", table=\"index_filename_troubleshooting_hos_bucket\", type=\"index\"}" timestamp=1689545828808
Jul 17 03:18:09 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:09.539Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_storeFileCount\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"default\", project=\"OLAP\", project_id=\"3\", region=\"2e1f4bf7efca655438213e8ba2c8c3a7\", table=\"index_partfile_default\", type=\"index\"}" timestamp=1689545828808
Jul 17 03:18:09 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:09.931Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_getCount\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", module=\"DC-HBase\", module_id=\"3\", namespace=\"SYSTEM\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"05db796ef377dd930b460cc22061d03b\", table=\"LOG\", type=\"data\"}" timestamp=1689545828808
Jul 17 03:18:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:10.346Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_STATS_metric_flushOutputSize_99th_percentile\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545828808
Jul 17 03:18:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:10.423Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_STATS_metric_compactedOutputBytes\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545828808
Jul 17 03:18:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:10.598Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_CompactionTime_95th_percentile\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Server\"}" timestamp=1689545828808
Jul 17 03:18:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:10.638Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_filteredReadRequestCount\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"rtp_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"d69626260aea1db953fc4fd4c9493c3c\", table=\"index_filename_rtp_hos_bucket\", type=\"index\"}" timestamp=1689545828808
Jul 17 03:18:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:10.754Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_avgStoreFileAge\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"dfa2d065ee026585e9af86e1e567c425\", table=\"firewall_hos_bucket\", type=\"data\"}" timestamp=1689545828808
Jul 17 03:18:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:11.399Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_numFilesCompactedCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"knowledge_base_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"d7545eddd6684b532e594f03edaf2b47\", table=\"index_time_knowledge_base_hos_bucket\", type=\"index\"}" timestamp=1689545830148
Jul 17 03:18:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:11.564Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_PostIndexUpdateTime_min\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"PhoenixIndexer\"}" timestamp=1689545830148
Jul 17 03:18:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:11.639Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_193_1_metric_majorCompactionOutputSize_num_ops\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545830148
Jul 17 03:18:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:11.654Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_FlushOutputSize_25th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Server\"}" timestamp=1689545830148
Jul 17 03:18:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:11.706Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_session_record_hos_bucket_metric_scanTime_95th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545830148
Jul 17 03:18:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:12.261Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_session_record_hos_bucket_metric_majorCompactionInputFileCount_num_ops\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545830148
Jul 17 03:18:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:12.394Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_LOG_metric_incrementTime_min\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545830148
Jul 17 03:18:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:12.639Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_numFilesCompactedCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"firewall_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"de3e9af7463fe090b6840184e6339540\", table=\"index_time_firewall_hos_bucket\", type=\"index\"}" timestamp=1689545830148
Jul 17 03:18:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:12.796Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_hwmon_temp_crit_celsius\", asset=\"twa-dellr740-dt002\", asset_id=\"39\", chip=\"platform_coretemp_0\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"OLAP-Node-Exporter-twa-dellr740-dt002\", endpoint_id=\"259\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\", sensor=\"temp6\"}" timestamp=1689545832060
Jul 17 03:18:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:15.249Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"68\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545821230
Jul 17 03:18:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:15.371Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.grpc.v1.cri-sandboxes-2e122063ee2c201eace04f25ac4f6d02502e3258d1a3613479961720dee5e2b1-shm.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689545827046
Jul 17 03:18:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:15.410Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_tasks_max\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"vmtoolsd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689545827046
Jul 17 03:18:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:15.652Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"96\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545821230
Jul 17 03:18:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:15.668Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_timex_pps_stability_exceeded_total\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545821230
Jul 17 03:18:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:15.977Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"1\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545821230
Jul 17 03:18:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:16.012Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_size_bytes\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"shm\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/3eb3a553bd7e6029fbb78c8179cc54167632fb6c9f35257a375e0bace33c51ee/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545821230
Jul 17 03:18:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:16.023Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"66\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545821230
Jul 17 03:18:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:16.047Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"114\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545821230
Jul 17 03:18:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:16.131Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-d5fdb9f7a199cdf3bbfc06b2d761364c7bcda5d0af25b97ff2276bb33e7b8ece.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"scope\"}" timestamp=1689545827046
Jul 17 03:18:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:16.213Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"var-lib-kubelet-pods-704b5479\\\\x2db3a5\\\\x2d4f40\\\\x2da111\\\\x2d190391574a21-volume\\\\x2dsubpaths-config\\\\x2dvolume-telegraf\\\\x2dsecurity-2.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689545827046
Jul 17 03:18:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:16.300Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"0\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545821230
Jul 17 03:18:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:16.378Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"33\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545821230
Jul 17 03:18:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:16.434Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"110\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545821230
Jul 17 03:18:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:16.448Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"113\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545821230
Jul 17 03:18:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:16.468Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"51\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545821230
Jul 17 03:18:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:16.501Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"mrzcpd_hugepages_setup.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"service\"}" timestamp=1689545827046
Jul 17 03:18:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:19.527Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_scrape_collector_duration_seconds\", asset=\"twa-tsgx06\", asset_id=\"101\", collector=\"filesystem\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545837612
Jul 17 03:18:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:19.540Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_scrape_collector_duration_seconds\", asset=\"twa-tsgx06\", asset_id=\"101\", collector=\"timex\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545837612
Jul 17 03:18:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:19.688Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"var-lib-kubelet-pods-7fc8eb18\\\\x2ddf9b\\\\x2d4e31\\\\x2daa08\\\\x2de338eee64b81-volume\\\\x2dsubpaths-telegraf-telegraf\\\\x2dsecurity-0.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689545824356
Jul 17 03:18:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:20.182Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_info\", asset=\"twa-tsgx06\", asset_id=\"101\", configmap=\"kubernetes-dashboard-settings\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545833949
Jul 17 03:18:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:20.258Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_waiting\", asset=\"twa-tsgx06\", asset_id=\"101\", container=\"helm\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-traefik-l5xzw\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"11a6aee2-f492-41ef-931d-e00b56122be3\"}" timestamp=1689545833949
Jul 17 03:18:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:20.276Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"28\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545837612
Jul 17 03:18:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:20.299Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_colls_total\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth814db6c2\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545837612
Jul 17 03:18:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:20.333Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"nsfs\", name=\"run-netns-cni\\\\x2d6eb58989\\\\x2d5d99\\\\x2d2407\\\\x2df0ab\\\\x2db63bd4c4de02.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689545824356
Jul 17 03:18:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:20.589Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_complete\", asset=\"twa-tsgx06\", asset_id=\"101\", condition=\"false\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", job_name=\"helm-install-tsg-traffic-engine-numa-0\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545833949
Jul 17 03:18:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:20.707Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_net_dev_group\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens2f1np1\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545837612
Jul 17 03:18:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:20.762Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_edac_csrow_correctable_errors_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", controller=\"0\", csrow=\"unknown\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545828263
Jul 17 03:18:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:20.775Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_net_dev_group\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth946c4097\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545837612
Jul 17 03:18:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:20.788Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_size_bytes\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda5\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", fstype=\"ext4\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/home\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545828263
Jul 17 03:18:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:20.790Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", container=\"coredns\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"running\"}" timestamp=1689545823249
Jul 17 03:18:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:20.850Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_annotations\", asset=\"pcap-tsgx10\", asset_id=\"89\", configmap=\"extension-apiserver-authentication\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545833828
Jul 17 03:18:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:20.883Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_ulimits_soft\", asset=\"twa-tsgx06\", asset_id=\"101\", container=\"telegraf\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", ulimit=\"max_open_files\"}" timestamp=1689545829652
Jul 17 03:18:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:20.888Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"timers.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"target\"}" timestamp=1689545830657
Jul 17 03:18:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:20.909Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"70\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545837612
Jul 17 03:18:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:20.933Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-baf143ea662db4696fc6be8e7e1e5cd8ca117bde89710c2a27a29754e4e35cd5.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"scope\"}" timestamp=1689545824356
Jul 17 03:18:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:20.943Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_waiting\", asset=\"twa-tsgx06\", asset_id=\"101\", container=\"exporter\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"systemd-exporter-7f549d9485-d9qg4\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"5efb71d9-5988-4e2b-a058-7799e16a5dfa\"}" timestamp=1689545833949
Jul 17 03:18:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:20.954Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"lldpad.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"simple\", type=\"service\"}" timestamp=1689545830657
Jul 17 03:18:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:20.971Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"53\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545837612
Jul 17 03:18:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:20.971Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-617272f1\\\\x2de26a\\\\x2d4d41\\\\x2db763\\\\x2d554ea7e04473-volume\\\\x2dsubpaths-config\\\\x2dvolume-firewall-1.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689545830657
Jul 17 03:18:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:20.986Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_info\", asset=\"twa-tsgx06\", asset_id=\"101\", configmap=\"coredns\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545833949
Jul 17 03:18:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:20.989Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db64\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545831998
Jul 17 03:18:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:21.028Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_fifo_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens3f0np0\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545828263
Jul 17 03:18:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:21.042Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_packets_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"lo\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545828263
Jul 17 03:18:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:21.051Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"7\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545837612
Jul 17 03:18:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:21.058Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db161\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545834840
Jul 17 03:18:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:21.206Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_up_changes_total\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"virtio_dign_c\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545837612
Jul 17 03:18:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:21.253Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_errs_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"vethc4731574\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545828263
Jul 17 03:18:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:21.264Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_errs_total\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth0ae4eb9d\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545837612
Jul 17 03:18:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:21.268Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_commands_processed_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545834840
Jul 17 03:18:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:21.276Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_commands_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cmd=\"del\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545831998
Jul 17 03:18:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:21.681Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-ttyS2.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689545830657
Jul 17 03:18:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:21.703Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Send_Udp\", app_name=\"sapp4\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", vsysId=\"1\"}" timestamp=1689545838534
Jul 17 03:18:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:21.713Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-617272f1\\\\x2de26a\\\\x2d4d41\\\\x2db763\\\\x2d554ea7e04473-volume\\\\x2dsubpaths-telegraf-telegraf-0.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689545830657
Jul 17 03:18:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:21.719Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-ask-password-wall.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689545830657
Jul 17 03:18:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:21.799Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"multipathd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"service\"}" timestamp=1689545830657
Jul 17 03:18:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:21.983Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_packets_received_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"mlx5_4\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545828263
Jul 17 03:18:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:22.699Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_buck_hash_sys_bytes\", asset=\"msh-dellr740-olap004\", asset_id=\"17\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Spark-msh-dellr740-olap004\", endpoint_id=\"15\", module=\"NC-Spark\", module_id=\"47\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545842256
Jul 17 03:18:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:23.616Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"msh-dellr740-olap004\", asset_id=\"17\", cpu=\"1\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Spark-msh-dellr740-olap004\", endpoint_id=\"15\", module=\"NC-Spark\", module_id=\"47\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545842256
Jul 17 03:18:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:23.684Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_writes_bytes_total\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689545832261
Jul 17 03:18:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:23.706Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_cpu_seconds_total\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"auditd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689545838416
Jul 17 03:18:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:24.199Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_reads_total\", asset=\"msh-tsgx09\", asset_id=\"55\", container=\"telegraf-security\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689545835306
Jul 17 03:18:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:24.245Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_bytes_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", interface=\"net1\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545837721
Jul 17 03:18:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:24.263Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_drop_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"eno33np0\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545840160
Jul 17 03:18:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:24.335Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-9a27d2386e66e0af456c0e62227159bd398f238e8795349cb024a04730a86c2d.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"scope\"}" timestamp=1689545838416
Jul 17 03:18:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:24.354Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-badc6d80be8fc6a17c57d479d7af9ac71f403d1b8ddf9fa49ecca8350872cc4f-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689545838416
Jul 17 03:18:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:24.373Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"41\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545840160
Jul 17 03:18:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:24.442Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_status_failed\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", job_name=\"helm-install-prometheus\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545840068
Jul 17 03:18:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:24.451Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db248\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545841795
Jul 17 03:18:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:24.479Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_master_link_up\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", master_host=\"192.168.20.157\", master_port=\"7112\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545841795
Jul 17 03:18:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:24.528Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_labels\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-systemd-exporter-zg78t\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"b4187347-4485-4cc4-92e8-53ea7fe72a5f\"}" timestamp=1689545831122
Jul 17 03:18:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:24.544Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_labels\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-server\", uid=\"5d69a0e6-abb0-4748-927e-25c8b27a7f8d\"}" timestamp=1689545831122
Jul 17 03:18:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:24.576Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-ttyS1.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689545831102
Jul 17 03:18:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:24.723Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_working_set_bytes\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"prometheus-alertmanager\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545838753
Jul 17 03:18:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:24.724Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"process_virtual_memory_max_bytes\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"ipmi-exporter\"}" timestamp=1689545834325
Jul 17 03:18:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:24.773Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failcnt\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"prometheus-node-exporter\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545825142
Jul 17 03:18:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:24.784Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"104\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545840160
Jul 17 03:18:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:24.791Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"firewall\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", failure_type=\"pgmajfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"container\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689545840041
Jul 17 03:18:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:24.813Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link_mode\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens2f1np1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545840160
Jul 17 03:18:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:24.823Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_info\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"ipmi-exporter\", version=\"go1.13.10\"}" timestamp=1689545834325
Jul 17 03:18:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:24.826Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"21\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545840160
Jul 17 03:18:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:24.869Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_resident_memory_bytes\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"tsg-clixon-backend.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689545831102
Jul 17 03:18:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:24.896Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-update-utmp.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"service\"}" timestamp=1689545831102
Jul 17 03:18:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:24.899Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_bytes_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"eno33np0\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545840160
Jul 17 03:18:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:24.910Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"ext4\", name=\"data-var-lib-kubelet-pods-692f1a13\\\\x2d48e0\\\\x2d47b7\\\\x2dad46\\\\x2da5b348bd402b-volume\\\\x2dsubpaths-telegraf-telegraf\\\\x2dsecurity-0.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689545831102
Jul 17 03:18:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:24.912Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"var-lib-kubelet-pods-5724459f\\\\x2d9a6b\\\\x2d46d1\\\\x2d9437\\\\x2db9002b60d530-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2dg5s6d.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689545831102
Jul 17 03:18:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:24.936Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kmod-static-nodes.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"service\"}" timestamp=1689545831102
Jul 17 03:18:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:24.945Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"msh-tsgx09\", asset_id=\"55\", condition=\"false\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-tsg-traffic-engine-numa-0-n2fbk\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"044ab283-1226-4087-a5d1-3cb614606766\"}" timestamp=1689545840068
Jul 17 03:18:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:25.002Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_labels\", asset=\"msh-tsgx09\", asset_id=\"55\", configmap=\"sapp-tsg-traffic-engine-numa-1\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545840068
Jul 17 03:18:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:25.041Z caller=dedupe.go:112 component=remote level=error remote_name=52db6f url=http://10.10.10.159:10090/nz-agent/prometheus/proxy/api/v1/write msg="non-recoverable error" count=1000 exemplarCount=0 err="server returned HTTP status 400 : out of order sample"
Jul 17 03:18:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:25.129Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_owner\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", owner_is_controller=\"true\", owner_kind=\"ReplicaSet\", owner_name=\"prometheus-kube-state-metrics-6b768c7bf8\", pod=\"prometheus-kube-state-metrics-6b768c7bf8-nfws4\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"99e215df-15ea-44bd-824d-42fb75c6e74b\"}" timestamp=1689545831122
Jul 17 03:18:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:25.146Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_labels\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"chart-values-ipmi-exporter\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545831122
Jul 17 03:18:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:25.167Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"redis\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Discard\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545824189
Jul 17 03:18:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:25.194Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_drop_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth5d71a3ad\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545840160
Jul 17 03:18:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:25.203Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas_ready\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"kubernetes-dashboard\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545831122
Jul 17 03:18:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:25.213Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_max_fds\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"rdma-ndd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689545831102
Jul 17 03:18:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:25.219Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", phase=\"Running\", pod=\"prometheus-node-exporter-qmjq7\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"e8a13065-a344-44f4-a0e1-7369c47522e2\"}" timestamp=1689545831122
Jul 17 03:18:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:25.236Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"tmpfs\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/data/var/lib/kubelet/pods/cbd312de-6144-4424-8813-fba2b3b16e9b/volumes/kubernetes.io~projected/kube-api-access-km9bh\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545840160
Jul 17 03:18:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:25.337Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_running\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"metrics-server\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"metrics-server-54dc485875-fkhsg\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"8133a051-391d-44a4-bc3f-8749cb1b67a5\"}" timestamp=1689545831122
Jul 17 03:18:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:25.381Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-infiniband-umad1.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689545831102
Jul 17 03:18:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:25.447Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_packets_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", interface=\"ens2f0np0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545841719
Jul 17 03:18:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:25.475Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"telegraf\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", state=\"uninterruptible\"}" timestamp=1689545826836
Jul 17 03:18:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:25.491Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"24\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545840160
Jul 17 03:18:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:25.505Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"81\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545840160
Jul 17 03:18:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:26.033Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"85\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545835810
Jul 17 03:18:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:26.073Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_hwmon_temp_celsius\", asset=\"twa-tsgx09\", asset_id=\"107\", chip=\"pci0000:00_0000:00:18_3\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", sensor=\"temp8\", service=\"prometheus-node-exporter\"}" timestamp=1689545835810
Jul 17 03:18:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:26.087Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"8\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545835810
Jul 17 03:18:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:26.188Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"56\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545835810
Jul 17 03:18:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:26.224Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"16\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545835810
Jul 17 03:18:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:26.262Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_commands_duration_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cmd=\"ping\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545843549
Jul 17 03:18:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:26.672Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"20\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545835810
Jul 17 03:18:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:26.694Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_files\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"shm\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/f372bdd609e3ce8b668b92a1a1b652a05916d01e3eac6943fef3777b6fc0431d/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545835810
Jul 17 03:18:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:26.935Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545835810
Jul 17 03:18:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:27.094Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_netstat_Icmp_InMsgs\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545835810
Jul 17 03:18:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:27.228Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_restart_policy\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-tsg-traffic-engine-numa-0-85495\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", type=\"OnFailure\", uid=\"32092806-1b0c-44f8-8750-8d654c41d7a9\"}" timestamp=1689545838744
Jul 17 03:18:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:27.243Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_running\", asset=\"twa-tsgx09\", asset_id=\"107\", container=\"prometheus-server\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-server-6ff784d758-mtn8x\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"55c0a8e2-51a8-48bf-a3cc-51508b7d7254\"}" timestamp=1689545838744
Jul 17 03:18:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:27.263Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_context_switches_total\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545835810
Jul 17 03:18:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:27.277Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"getty.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"target\"}" timestamp=1689545833907
Jul 17 03:18:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:27.397Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_multicast_packets_transmitted_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_5\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545842882
Jul 17 03:18:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:27.414Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_colls_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens2f1np1\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545842882
Jul 17 03:18:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:27.453Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"18\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545835810
Jul 17 03:18:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:27.535Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"pcap-kvm-cm001\", asset_id=\"45\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db15\", endpoint=\"CM-Redis-pcap-kvm-cm001\", endpoint_id=\"305\", host=\"192.168.10.157\", module=\"CM-Redis\", module_id=\"65\", nz_agent_id=\"3\", parent_asset=\"pcap-dellr740-pve001\", parent_asset_id=\"43\", project=\"Central-Management\", project_id=\"5\"}" timestamp=1689545846191
Jul 17 03:18:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:27.594Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"var-lib-kubelet-pods-c8850d1a\\\\x2d9bcd\\\\x2d4bf0\\\\x2dadad\\\\x2d91f75a60d9cc-volumes-kubernetes.io\\\\x7esecret-kubernetes\\\\x2ddashboard\\\\x2dcerts.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689545833907
Jul 17 03:18:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:27.614Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"32\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545842882
Jul 17 03:18:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:27.637Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_status_completion_time\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", job_name=\"helm-install-systemd-exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545835682
Jul 17 03:18:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:27.637Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"3\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545842882
Jul 17 03:18:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:27.641Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda5\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", fstype=\"ext4\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/var/lib/kubelet/pods/0b0292d1-7c6a-4fad-a1f9-f14e3c7cb4f8/volume-subpaths/config-volume/telegraf-security/1\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545842882
Jul 17 03:18:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:27.653Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"msh-tsgx10\", asset_id=\"69\", condition=\"true\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"kube-multus-ds-ntb9l\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"396cb6af-9b76-4a32-aad2-deb6dbe16f5b\"}" timestamp=1689545835682
Jul 17 03:18:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:27.656Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"ext4\", name=\"data-var-lib-kubelet-pods-848571bc\\\\x2da31e\\\\x2d44de\\\\x2da0f0\\\\x2dd172cf8f3bd0-volume\\\\x2dsubpaths-config\\\\x2dvolume-telegraf-2.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689545834114
Jul 17 03:18:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:27.663Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_id\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth6f518146\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545842882
Jul 17 03:18:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:27.891Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_created\", asset=\"msh-tsgx10\", asset_id=\"69\", configmap=\"kube-root-ca.crt\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545835682
Jul 17 03:18:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:27.987Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_spec_completions\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", job_name=\"helm-install-prometheus\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545838744
Jul 17 03:18:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:28.048Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_namespace_created\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545835682
Jul 17 03:18:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:28.053Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_ulimits_soft\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"exporter-merger\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", ulimit=\"max_open_files\"}" timestamp=1689545841124
Jul 17 03:18:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:28.067Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"52\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545842882
Jul 17 03:18:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:28.072Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_metadata_generation\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"tsg-traffic-engine-numa-0-firewall\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545835682
Jul 17 03:18:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:28.090Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_resident_memory_bytes\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"mrzcpd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689545834114
Jul 17 03:18:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:28.124Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"38\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545842882
Jul 17 03:18:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:28.149Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2dpartuuid-1fa48360\\\\x2de7f3\\\\x2d4756\\\\x2db3f3\\\\x2dee1646b57230.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689545834114
Jul 17 03:18:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:28.215Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545842882
Jul 17 03:18:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:28.249Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545842882
Jul 17 03:18:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:28.282Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"114\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545842882
Jul 17 03:18:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:28.397Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"ext4\", name=\"var-lib-kubelet-pods-b60c5b05\\\\x2d6d97\\\\x2d4ae4\\\\x2dbf4c\\\\x2d866d931e73ec-volume\\\\x2dsubpaths-config\\\\x2dpath-dign\\\\x2dclient-3.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689545834114
Jul 17 03:18:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:28.461Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-pod0b0292d1_7c6a_4fad_a1f9_f14e3c7cb4f8.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"slice\"}" timestamp=1689545834114
Jul 17 03:18:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:28.738Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_processes\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"prometheus-node-exporter\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545834747
Jul 17 03:18:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:31.205Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_start_time\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"coredns-696d987597-cgfrx\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"84148aa3-78e1-49bc-9b4c-b596d7dd94b8\"}" timestamp=1689545844068
Jul 17 03:18:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:31.219Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_annotations\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"systemd-exporter-7f549d9485\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545844068
Jul 17 03:18:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:31.484Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"24\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545840939
Jul 17 03:18:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:31.500Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_net_dev_group\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens4f3np3\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545839093
Jul 17 03:18:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:31.578Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"msh-tsgx05\", asset_id=\"67\", container=\"coredns\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"running\"}" timestamp=1689545835062
Jul 17 03:18:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:31.584Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"msh-tsgx05\", asset_id=\"67\", condition=\"unknown\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-mrzcpd-exporter-8dssx\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"24d38c73-ce01-4d33-ae4e-dc1ddfa73fb6\"}" timestamp=1689545844068
Jul 17 03:18:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:31.598Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"74\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545840939
Jul 17 03:18:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:31.615Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"2\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545840939
Jul 17 03:18:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:31.616Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"blk-availability.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"service\"}" timestamp=1689545841015
Jul 17 03:18:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:31.638Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_info\", asset=\"msh-tsgx05\", asset_id=\"67\", board_id=\"MT_0000000359\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_1\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", firmware_version=\"22.28.4000\", hca_type=\"MT4125\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545840939
Jul 17 03:18:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:31.663Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"msh-tsgx05\", asset_id=\"67\", condition=\"false\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"dign-client-469mx\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"9aea822c-d5bb-4081-ad0f-a7aa2fc38dfc\"}" timestamp=1689545844068
Jul 17 03:18:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:31.698Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_created\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", job_name=\"helm-install-mrzcpd-exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545844068
Jul 17 03:18:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:31.703Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_id\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"vetheef6d56c\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545840791
Jul 17 03:18:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:31.717Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx05\", asset_id=\"67\", container=\"configmap-reloader\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Discard\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545819494
Jul 17 03:18:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:31.729Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_receive_switch_relay_errors_total\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545840791
Jul 17 03:18:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:31.732Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_restarts_total\", asset=\"msh-tsgx05\", asset_id=\"67\", container=\"helm\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-tsg-traffic-engine-numa-1-lv7qp\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"aa621a66-69cd-4fef-a816-ab5978f42426\"}" timestamp=1689545844068
Jul 17 03:18:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:31.748Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"45\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545840939
Jul 17 03:18:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:31.760Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545839093
Jul 17 03:18:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:31.767Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-1ef773a86a4a57c19fc87996aed59dd63c624d5fae04b097728a17b29ac88dbb.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"scope\"}" timestamp=1689545842644
Jul 17 03:18:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:31.778Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_slowlog_last_id\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545836711
Jul 17 03:18:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:31.804Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_gc_cpu_fraction\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"ipmi-exporter\"}" timestamp=1689545840809
Jul 17 03:18:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:31.823Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_daemonset_status_number_ready\", asset=\"msh-tsgx05\", asset_id=\"67\", daemonset=\"dign-client\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545844068
Jul 17 03:18:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:31.837Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"user.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"slice\"}" timestamp=1689545841015
Jul 17 03:18:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:31.844Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_load_average_10s\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"exporter\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545834337
Jul 17 03:18:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:31.859Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_resident_memory_bytes\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"tsg-clixon-restconf.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689545842644
Jul 17 03:18:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:31.936Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_colls_total\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"vethd59521a8\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545840791
Jul 17 03:18:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:32.079Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx05\", asset_id=\"67\", container=\"configmap-reloader\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Write\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545819494
Jul 17 03:18:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:32.085Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"122\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545840939
Jul 17 03:18:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:32.091Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"10\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545839093
Jul 17 03:18:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:32.116Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx05\", asset_id=\"67\", container=\"lb-tcp-80\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Read\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545823654
Jul 17 03:18:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:32.129Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_labels\", asset=\"msh-tsgx05\", asset_id=\"67\", configmap=\"chart-content-traefik\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545844068
Jul 17 03:18:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:32.129Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_ready_replicas\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-1-firewall-6d8d5654d\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545839276
Jul 17 03:18:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:32.145Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_created\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-alertmanager-6df7f457b6-xnxfq\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"b7e44624-9b20-4189-9ac3-b8a15c1776a6\"}" timestamp=1689545839276
Jul 17 03:18:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:32.149Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-netns-cni\\\\x2daa548524\\\\x2d41c9\\\\x2d5b77\\\\x2d3e83\\\\x2d1f16304a4188.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689545842644
Jul 17 03:18:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:32.188Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"21\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545840791
Jul 17 03:18:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:32.210Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_files_free\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda5\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", fstype=\"ext4\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/var\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545840791
Jul 17 03:18:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:32.222Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db176\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545836711
Jul 17 03:18:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:32.234Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_up_changes_total\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"eno34np1\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545839093
Jul 17 03:18:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:32.400Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_timex_pps_stability_exceeded_total\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545839093
Jul 17 03:18:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:32.420Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_commands_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cmd=\"del\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545846438
Jul 17 03:18:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:32.480Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_changes_total\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens6f1np1\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545840605
Jul 17 03:18:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:32.621Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"59\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545839093
Jul 17 03:18:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:32.749Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"105\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545840791
Jul 17 03:18:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:32.840Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas_updated\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"tsg-traffic-engine-numa-0-firewall\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545839894
Jul 17 03:18:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:32.922Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_virtual_memory_bytes\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"rtkit-daemon.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689545842644
Jul 17 03:18:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:33.129Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_bytes_total\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens8f1np1\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545840605
Jul 17 03:18:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:33.156Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"29\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545840605
Jul 17 03:18:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:33.194Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_netstat_Udp6_RcvbufErrors\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545840605
Jul 17 03:18:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:33.221Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-c36bbd943b007854126bb5e121d9774579cbfee4d4666edddd5c6b1038f9db35.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"scope\"}" timestamp=1689545847691
Jul 17 03:18:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:33.514Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas_available\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"configmap-reloader\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545839894
Jul 17 03:18:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:33.608Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_terminated\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"helm\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-tsg-traffic-engine-numa-0-p9rhf\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"105764ab-9632-4df2-829e-d2392e8ab613\"}" timestamp=1689545839276
Jul 17 03:18:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:33.655Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_created\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_endpoint=\"cm-policy-local-cache-redis-master\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545839894
Jul 17 03:18:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:33.962Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-e4ba4b14a19c466b5a530124fe01c796f501f2cbf6ffef19efc3407f0bac587d.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"scope\"}" timestamp=1689545847691
Jul 17 03:18:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:33.997Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_ready_replicas\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-1-firewall-fbc6788df\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545839894
Jul 17 03:18:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:34.025Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_protocol_type\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth989c1124\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545840605
Jul 17 03:18:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:34.044Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-37526140\\\\x2d01d6\\\\x2d46f5\\\\x2d9f0b\\\\x2de066a6254bac-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2dcftdn.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689545847691
Jul 17 03:18:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:40.376Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_drop_total\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"br_dign_c\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545851647
Jul 17 03:18:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:40.388Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"121\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545851647
Jul 17 03:18:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:40.465Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Total\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689545830489
Jul 17 03:18:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:40.479Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failcnt\", asset=\"twa-tsgx04\", asset_id=\"97\", container=\"monit-stream\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545844333
Jul 17 03:18:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:40.492Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"0\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545851647
Jul 17 03:18:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:40.732Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_rdb_bgsave_in_progress\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545853827
Jul 17 03:18:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:40.746Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_compactionsFailedCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"firewall_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"0abad7a566d328d4b2672f660372310f\", table=\"index_filename_firewall_hos_bucket\", type=\"index\"}" timestamp=1689545859757
Jul 17 03:18:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:40.842Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_compactionsCompletedCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"firewall_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"57210503f594ede560ea641f752e077d\", table=\"index_filename_firewall_hos_bucket\", type=\"index\"}" timestamp=1689545859757
Jul 17 03:18:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:40.854Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx04\", asset_id=\"97\", container=\"dashboard-metrics-scraper\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Discard\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545841867
Jul 17 03:18:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:40.874Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2dpartlabel-TSG\\\\x2dOS\\\\x2dUPDATE.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689545855764
Jul 17 03:18:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:41.052Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_LOG_metric_deleteBatchTime_mean\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545859757
Jul 17 03:18:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:41.067Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_deleteCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"f1a9d7a24697aa98a6aa936c2dddd33e\", table=\"troubleshooting_hos_bucket\", type=\"data\"}" timestamp=1689545859757
Jul 17 03:18:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:41.149Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_maxFlushQueueSize\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"d0e4dc2b30bbf3856137f72fa5acc597\", table=\"firewall_hos_bucket\", type=\"data\"}" timestamp=1689545859757
Jul 17 03:18:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:41.217Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_name_assign_type\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens7f1np1\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545851647
Jul 17 03:18:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:41.251Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-866c527e\\\\x2dd063\\\\x2d4cfb\\\\x2db039\\\\x2dfa190c69b128-volumes-kubernetes.io\\\\x7esecret-kubernetes\\\\x2ddashboard\\\\x2dcerts.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689545855764
Jul 17 03:18:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:41.348Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_10_193_1_metric_majorCompactionTime_mean\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545859757
Jul 17 03:18:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:41.511Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"twa-tsgx04\", asset_id=\"97\", condition=\"true\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", pod=\"tsg-traffic-engine-numa-1-firewall-7bff8557d6-cnnft\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"9ad11045-e0bd-4481-9c2d-4c81a67475ee\"}" timestamp=1689545860034
Jul 17 03:18:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:41.532Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_mapped_file\", asset=\"twa-tsgx04\", asset_id=\"97\", container=\"kube-multus\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545841850
Jul 17 03:18:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:41.537Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"cm-policy-local-cache-redis-master-69967b5879-j7kh6\", project=\"TSG-OS\", project_id=\"7\", reason=\"Evicted\", service=\"prometheus-kube-state-metrics\", uid=\"dc48faf7-72ec-4d62-807b-3dca8380f0aa\"}" timestamp=1689545860034
Jul 17 03:18:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:41.775Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"16\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545851647
Jul 17 03:18:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:41.847Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_TASK_metric_putBatchTime_median\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545859757
Jul 17 03:18:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:42.303Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_hbase_table_rsgroup_metric_compactionOutputFileCount_90th_percentile\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545859757
Jul 17 03:18:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:42.464Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_bytes_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"virtio_dign_s\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545856242
Jul 17 03:18:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:42.469Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_commands_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", cmd=\"incrby\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545854838
Jul 17 03:18:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:42.604Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-systemd-exporter-cbljh\", project=\"TSG-OS\", project_id=\"7\", reason=\"Shutdown\", service=\"prometheus-kube-state-metrics\", uid=\"da14c78d-f512-4a0f-b73a-808e63c92307\"}" timestamp=1689545855154
Jul 17 03:18:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:42.740Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Tcp_Bypass_Pkt\", app_name=\"sapp4\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", vsysId=\"1\"}" timestamp=1689545849178
Jul 17 03:18:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:42.762Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"lldpad.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689545860671
Jul 17 03:18:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:42.800Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-e12645fd77e0267b7e4793e719ced23b1c706f0f70bf9a2b3489baaad542935d.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"scope\"}" timestamp=1689545860671
Jul 17 03:18:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:42.842Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_10_194_1_metric_flushOutputSize_95th_percentile\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545859757
Jul 17 03:18:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:42.921Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", cpu=\"46\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Gohangout-msh-dellr740-dt002\", endpoint_id=\"167\", mode=\"user\", module=\"DC-Gohangout\", module_id=\"35\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545862623
Jul 17 03:18:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:43.448Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", cpu=\"26\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Gohangout-msh-dellr740-dt002\", endpoint_id=\"167\", mode=\"iowait\", module=\"DC-Gohangout\", module_id=\"35\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545862623
Jul 17 03:18:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:43.764Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", container=\"firewall\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", state=\"iowaiting\"}" timestamp=1689545860941
Jul 17 03:18:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:43.943Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_running\", asset=\"pcap-tsgx01\", asset_id=\"71\", container=\"prometheus-alertmanager-configmap-reload\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-alertmanager-6df7f457b6-m8cl7\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"98daa3ea-45f0-41c2-be94-512fb3e2a016\"}" timestamp=1689545855154
Jul 17 03:18:44 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:44.021Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failcnt\", asset=\"pcap-tsgx01\", asset_id=\"71\", container=\"firewall\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689545858664
Jul 17 03:18:44 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:44.040Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"var-lib-kubelet-pods-50480520\\\\x2d32d8\\\\x2d413b\\\\x2d9354\\\\x2dfde3e50919a7-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2dsv7tl.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689545860671
Jul 17 03:18:44 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:44.060Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_owner\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", job_name=\"helm-install-cm-policy-local-cache\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", owner_is_controller=\"<none>\", owner_kind=\"<none>\", owner_name=\"<none>\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545855154
Jul 17 03:18:44 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:44.117Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", cpu=\"124\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545856242
Jul 17 03:18:44 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:44.133Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", cpu=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545856242
Jul 17 03:18:44 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:44.141Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_ready\", asset=\"pcap-tsgx01\", asset_id=\"71\", container=\"redis\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"cm-policy-local-cache-redis-master-69967b5879-6px4d\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"8881282f-a626-46c9-8a8a-2eedf6fae182\"}" timestamp=1689545855154
Jul 17 03:18:46 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:46.018Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_last_gc_time_seconds\", asset=\"msh-kvm-cm001\", asset_id=\"3\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"CM-Node-Exporter-msh-kvm-cm001\", endpoint_id=\"391\", module=\"CM-Node-Exporter\", module_id=\"75\", nz_agent_id=\"3\", parent_asset=\"msh-dellr740-pve001\", parent_asset_id=\"1\", project=\"Central-Management\", project_id=\"5\"}" timestamp=1689545864235
Jul 17 03:18:47 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:47.022Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_HlogSplitSize_99th_percentile\", asset=\"pcap-dellr740-dt003\", asset_id=\"35\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt003-Hmaster\", endpoint_id=\"387\", module=\"DC-HBase\", module_id=\"3\", name=\"Master\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"FileSystem\"}" timestamp=1689545866370
Jul 17 03:18:48 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:48.606Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_numReferenceFiles\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"session_record_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"88a9bbe2136852f7ad790e6c839f28d2\", table=\"index_time_session_record_hos_bucket\", type=\"index\"}" timestamp=1689545867438
Jul 17 03:18:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:49.250Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_2_metric_flushMemstoreSize_num_ops\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545867438
Jul 17 03:18:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:49.435Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_avgStoreFileAge\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", namespace=\"SYSTEM\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"5a8f51f1ceb29e2c5e26a5988d4dce97\", table=\"LOG\", type=\"data\"}" timestamp=1689545867438
Jul 17 03:18:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:49.443Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_1_metric_deleteBatchTime_mean\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545867438
Jul 17 03:18:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:49.580Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_tsg_galaxy_table_relation_user_teid_metric_appendTime_min\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545867438
Jul 17 03:18:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:49.628Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_numBytesCompactedCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"troubleshooting_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"b301f632e0b40d5f825252a4bde96f7d\", table=\"index_filename_troubleshooting_hos_bucket\", type=\"index\"}" timestamp=1689545867438
Jul 17 03:18:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:49.805Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_sector_reads_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/data/var/lib/kubelet/pods/32707ee1-6f45-4d2c-8e0e-78edc233adc3/volumes/kubernetes.io~projected/kube-api-access-x64jh\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545858313
Jul 17 03:18:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:49.817Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_session_record_hos_bucket_metric_putTime_99_9th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545867438
Jul 17 03:18:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:49.843Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_1_metric_putBatchTime_max\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545867438
Jul 17 03:18:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:49.884Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_write_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/data/var/lib/kubelet/pods/1617bf89-68ca-42e6-8a6e-a79f76bf18ce/volumes/kubernetes.io~projected/kube-api-access-hjpfx\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545858313
Jul 17 03:18:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:49.902Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_complete\", asset=\"twa-tsgx08\", asset_id=\"105\", condition=\"unknown\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", job_name=\"helm-install-systemd-exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545861568
Jul 17 03:18:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:49.925Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_info\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", job_name=\"helm-delete-tsg-traffic-engine-numa-1\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545861568
Jul 17 03:18:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:49.978Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_3_metric_incrementTime_90th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545867438
Jul 17 03:18:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:50.014Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"twa-tsgx10\", asset_id=\"109\", cpu=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545861797
Jul 17 03:18:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:50.043Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_disk_reads_completed_total\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"sda\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-olap003\", endpoint_id=\"235\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"yarn_resourcemanager,zookeeper,nacos,kafka,spark,druid_data,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545869713
Jul 17 03:18:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:50.060Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_cpu_seconds_total\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"udisks2.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689545861057
Jul 17 03:18:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:50.071Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_cpu_shares\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"telegraf\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689545859050
Jul 17 03:18:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:50.078Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-podc11c351e_f86b_4fa5_8167_cadbe729e2c9.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"slice\"}" timestamp=1689545861057
Jul 17 03:18:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:50.095Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_daemonset_created\", asset=\"twa-tsgx10\", asset_id=\"109\", daemonset=\"dign-client\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545859668
Jul 17 03:18:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:50.098Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"smartd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"service\"}" timestamp=1689545861057
Jul 17 03:18:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:50.115Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_putCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"90681249af133b1efdb8e61bd7cd5c26\", table=\"default\", type=\"data\"}" timestamp=1689545867438
Jul 17 03:18:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:50.138Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_dormant\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethe417278f\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545861797
Jul 17 03:18:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:50.156Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_default_metric_storeCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545867438
Jul 17 03:18:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:50.180Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_1_metric_memStoreSize\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545867438
Jul 17 03:18:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:50.197Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_2_metric_majorCompactionInputSize_99_9th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545867438
Jul 17 03:18:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:50.234Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"tmpfs\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/var/lib/kubelet/pods/b95f152c-994f-41b5-9701-bd977efd71db/volumes/kubernetes.io~secret/config\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545861797
Jul 17 03:18:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:50.264Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", phase=\"Pending\", pod=\"systemd-exporter-7f549d9485-5b4m5\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"fe1b9683-356b-448d-8311-9cb2e5a90800\"}" timestamp=1689545861568
Jul 17 03:18:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:50.345Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link_mode\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"em1\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-olap003\", endpoint_id=\"235\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"yarn_resourcemanager,zookeeper,nacos,kafka,spark,druid_data,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545869713
Jul 17 03:18:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:50.401Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_created\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"ipmi-exporter\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545861568
Jul 17 03:18:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:50.421Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_dormant\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth25024717\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545866709
Jul 17 03:18:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:50.426Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Udp_Bypass_STM\", app_name=\"sapp4\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", vsysId=\"1\"}" timestamp=1689545868675
Jul 17 03:18:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:50.447Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"4\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545866709
Jul 17 03:18:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:50.463Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Pending\", pod=\"coredns-766944d45b-dstvr\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"b52de515-fecd-439f-b56f-e8953f782519\"}" timestamp=1689545859668
Jul 17 03:18:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:50.475Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"34\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545866709
Jul 17 03:18:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:50.491Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"76\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545861797
Jul 17 03:18:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:50.505Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:40-0000:40:01.1-0000:41:00.0-net-ens3f0np0.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689545866554
Jul 17 03:18:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:50.524Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"scrape_duration_seconds\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_instance=\"10.42.0.131:9273\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", vsysId=\"1\"}" timestamp=1689545865148
Jul 17 03:18:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:50.556Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_inodes_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"overlay_0-259\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545858313
Jul 17 03:18:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:50.558Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_owner\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", owner_is_controller=\"true\", owner_kind=\"Deployment\", owner_name=\"tsg-traffic-engine-numa-1-firewall\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-1-firewall-68fdf8b646\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545859668
Jul 17 03:18:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:50.570Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"40\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545866709
Jul 17 03:18:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:50.575Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx10\", asset_id=\"109\", cpu=\"40\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545861797
Jul 17 03:18:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:50.584Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_compressed_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth1e5d3348\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545866709
Jul 17 03:18:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:50.598Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"etc.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689545866554
Jul 17 03:18:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:50.603Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_cpu_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"atd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689545866554
Jul 17 03:18:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:50.604Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545866709
Jul 17 03:18:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:50.620Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:40-0000:40:01.1-0000:41:00.1-infiniband-mlx5_5.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689545866554
Jul 17 03:18:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:50.646Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_node_info\", asset=\"twa-tsgx10\", asset_id=\"109\", container_runtime_version=\"containerd://1.7.1-k3s1\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", internal_ip=\"10.10.30.170\", kernel_version=\"5.17.15-1.el8.x86_64\", kubelet_version=\"v1.27.2+k3s1\", kubeproxy_version=\"v1.27.2+k3s1\", module=\"TSG-OS\", module_id=\"77\", node=\"localhost\", nz_agent_id=\"3\", os_image=\"TSG-OS v23.07-rc1-079e964 (TSGXNXR620G40R01P0906)\", pod_cidr=\"10.42.0.0/24\", project=\"TSG-OS\", project_id=\"7\", provider_id=\"k3s://localhost\", service=\"prometheus-kube-state-metrics\", system_uuid=\"4c4c4544-0046-5410-8031-cac04f324733\"}" timestamp=1689545859668
Jul 17 03:18:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:50.656Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_labels\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-cm-policy-local-cache-rsnqh\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"f1dadae1-a545-435a-9bce-9a35d9db87be\"}" timestamp=1689545861568
Jul 17 03:18:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:50.701Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_carrier_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethd494b425\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545866709
Jul 17 03:18:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:50.736Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"tsg_master_mirror_pkts\", app_name=\"tsg_master\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"traffic-engine\", line_name=\"fail\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", table_name=\"flow\", vsysId=\"1\"}" timestamp=1689545868675
Jul 17 03:18:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:50.819Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_device_id\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"br_dign_c\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545861797
Jul 17 03:18:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:50.834Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"twa-tsgx10\", asset_id=\"109\", cpu=\"66\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545861797
Jul 17 03:18:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:50.843Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_writes_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"overlay_0-432\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545858313
Jul 17 03:18:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:50.859Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"twa-tsgx08\", asset_id=\"105\", container=\"firewall\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", state=\"running\"}" timestamp=1689545848717
Jul 17 03:18:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:50.876Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"102\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545866709
Jul 17 03:18:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:50.896Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-olap003\", endpoint_id=\"235\", module=\"OLAP-Node-Exporter\", module_id=\"51\", name=\"4\", nz_agent_id=\"3\", olap_node_exporter=\"yarn_resourcemanager,zookeeper,nacos,kafka,spark,druid_data,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\", type=\"Processor\"}" timestamp=1689545869713
Jul 17 03:18:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:50.988Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"phy_rx_packets_total\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", dev=\"ens1f0np0\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"mrzcpd-exporter\"}" timestamp=1689545855675
Jul 17 03:18:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:51.004Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db9\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545866625
Jul 17 03:18:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:51.016Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-virtual-net-vethec314a7d.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689545861057
Jul 17 03:18:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:51.021Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_restart_policy\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"coredns-55886d8f9d-vf9r8\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", type=\"Always\", uid=\"fd4b4291-55db-4894-9795-6abc7f3024db\"}" timestamp=1689545861568
Jul 17 03:18:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:52.420Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_scanCount\", asset=\"pcap-dellr740-dt002\", asset_id=\"33\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt002-HregionServer\", endpoint_id=\"155\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"1\", parent=\"knowledge_base_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"70b8c2c4e924bbd6fc1e96b790a6239d\", table=\"index_time_knowledge_base_hos_bucket\", type=\"index\"}" timestamp=1689545869685
Jul 17 03:18:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:52.458Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_limit_bytes\", asset=\"pcap-tsgx11\", asset_id=\"115\", container=\"prometheus-alertmanager-configmap-reload\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545870039
Jul 17 03:18:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:52.799Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", container=\"metrics-server\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"stopped\"}" timestamp=1689545869706
Jul 17 03:18:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:52.811Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", container=\"ipmi-exporter\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"sleeping\"}" timestamp=1689545868770
Jul 17 03:18:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:52.829Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_size_bytes\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"shm\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/bbab9a0f87816d9c49b20914f9611ba0f48dddfdac03dd0433d92497f69f5a9d/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545869337
Jul 17 03:18:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:53.065Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-kernel-config.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689545872135
Jul 17 03:18:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:53.070Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", cpu=\"76\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545869337
Jul 17 03:18:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:53.083Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_time_clocksource_available_info\", asset=\"pcap-tsgx11\", asset_id=\"115\", clocksource=\"acpi_pm\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"0\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545869337
Jul 17 03:18:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:53.130Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-server-6ff784d758-nghpf\", project=\"TSG-OS\", project_id=\"7\", reason=\"NodeAffinity\", service=\"prometheus-kube-state-metrics\", uid=\"21f91134-966a-424e-a89e-96c924c64f07\"}" timestamp=1689545864478
Jul 17 03:18:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:53.135Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_hwmon_temp_celsius\", asset=\"pcap-tsgx11\", asset_id=\"115\", chip=\"pci0000:00_0000:00:19_3\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", sensor=\"temp5\", service=\"prometheus-node-exporter\"}" timestamp=1689545869337
Jul 17 03:18:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:53.139Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_files_free\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"shm\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/6fd6f5dfbe9f4f64f473ffd2b9803a21e486bbbf7cfea44481b599ff4a7dc0f9/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545868328
Jul 17 03:18:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:53.151Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_flags\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"eno1\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545868328
Jul 17 03:18:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:53.152Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", cpu=\"116\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545869337
Jul 17 03:18:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:53.205Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"tmpfs\", name=\"var-lib-kubelet-pods-05cea850\\\\x2dfebe\\\\x2d4ad9\\\\x2da01e\\\\x2dde2d8c0e8a8b-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2dzgn8l.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689545865496
Jul 17 03:18:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:53.229Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db91\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545859367
Jul 17 03:18:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:53.409Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"msh-tsgx12\", asset_id=\"113\", container=\"dashboard-metrics-scraper\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"running\"}" timestamp=1689545847043
Jul 17 03:18:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:53.514Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx08\", asset_id=\"65\", cpu=\"73\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545868328
Jul 17 03:18:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:53.744Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-dellr740-dt002\", asset_id=\"33\", cpu=\"36\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"NC-Gohangout-pcap-dellr740-dt002\", endpoint_id=\"173\", mode=\"nice\", module=\"DC-Gohangout\", module_id=\"35\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545872666
Jul 17 03:18:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:53.893Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx12\", asset_id=\"113\", cpu=\"54\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545863391
Jul 17 03:18:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:53.916Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_protocol_type\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"eno36np3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545863391
Jul 17 03:18:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:53.938Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_compressed_total\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"virtio_dign_s\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545863391
Jul 17 03:18:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:53.986Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_reads_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", container=\"mrapm-device\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545861737
Jul 17 03:18:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:54.000Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_node_status_condition\", asset=\"pcap-tsgx07\", asset_id=\"85\", condition=\"DiskPressure\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", node=\"localhost\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"false\"}" timestamp=1689545862058
Jul 17 03:18:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:54.018Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"11\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545859409
Jul 17 03:18:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:54.023Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas_ready\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"prometheus-alertmanager\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545862058
Jul 17 03:18:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:54.040Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cpu=\"65\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545859409
Jul 17 03:18:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:54.050Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cpu=\"99\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545859409
Jul 17 03:18:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:54.071Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_info\", asset=\"pcap-tsgx07\", asset_id=\"85\", configmap=\"sapp-tsg-traffic-engine-numa-0\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545862058
Jul 17 03:18:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:54.087Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_bytes_total\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"cadvisor\", interface=\"eno2\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545855386
Jul 17 03:18:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:54.098Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_labels\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"mrzcpd-exporter\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545868950
Jul 17 03:18:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:54.105Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"msh-tsgx12\", asset_id=\"113\", container=\"prometheus-alertmanager\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"cadvisor\", failure_type=\"pgfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"hierarchy\"}" timestamp=1689545853541
Jul 17 03:18:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:54.116Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_owner\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", job_name=\"helm-install-mrzcpd-exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", owner_is_controller=\"<none>\", owner_kind=\"<none>\", owner_name=\"<none>\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545868950
Jul 17 03:18:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:54.150Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"101\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545859409
Jul 17 03:18:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:54.172Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx12\", asset_id=\"113\", cpu=\"39\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545863391
Jul 17 03:18:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:54.177Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_daemonset_status_updated_number_scheduled\", asset=\"msh-tsgx08\", asset_id=\"65\", daemonset=\"kube-multus-ds\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545868950
Jul 17 03:18:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:54.198Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_errs_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth8550139b\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545859409
Jul 17 03:18:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:54.201Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_netstat_Icmp6_InErrors\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545868328
Jul 17 03:18:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:54.222Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cpu=\"33\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545859409
Jul 17 03:18:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:54.229Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx12\", asset_id=\"113\", container=\"telegraf-security\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Read\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689545847700
Jul 17 03:18:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:54.264Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"br_dign_c\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545859409
Jul 17 03:18:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:54.295Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cpu=\"6\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545859409
Jul 17 03:18:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:54.309Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link_mode\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth149fb130\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545863391
Jul 17 03:18:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:54.335Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cpu=\"58\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545859409
Jul 17 03:18:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:54.338Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_metadata_resource_version\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"k3s-serving\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545862058
Jul 17 03:18:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:54.373Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_bytes_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens1f1np1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545859409
Jul 17 03:18:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:54.467Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-37750f9f\\\\x2d766e\\\\x2d4714\\\\x2da437\\\\x2dd6b2d6f2c90a-volume\\\\x2dsubpaths-config\\\\x2dvolume-firewall-1.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689545860884
Jul 17 03:18:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:54.468Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_hwmon_sensor_label\", asset=\"msh-tsgx12\", asset_id=\"113\", chip=\"pci0000:00_0000:00:18_3\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", label=\"tccd5\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", sensor=\"temp7\", service=\"prometheus-node-exporter\"}" timestamp=1689545863391
Jul 17 03:18:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:54.537Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"remote-fs-pre.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"target\"}" timestamp=1689545871179
Jul 17 03:18:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:54.658Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_max_usage_bytes\", asset=\"msh-tsgx08\", asset_id=\"65\", container=\"lb-tcp-80\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545849357
Jul 17 03:18:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:54.696Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-93e7480f89cd29bf696afedd9a4b230726c09369df1a638dd33fb6a7d0838e67.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"scope\"}" timestamp=1689545872135
Jul 17 03:18:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:54.868Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db62\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545870108
Jul 17 03:18:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:54.993Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_mapped_file\", asset=\"pcap-tsgx07\", asset_id=\"85\", container=\"dign-server-dns\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545862181
Jul 17 03:18:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:55.051Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"pcap-tsgx07\", asset_id=\"85\", condition=\"unknown\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", pod=\"tsg-traffic-engine-numa-0-firewall-5f84c6ddc7-42v4b\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"37750f9f-766e-4714-a437-d6b2d6f2c90a\"}" timestamp=1689545862058
Jul 17 03:18:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:55.148Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"34\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545863391
Jul 17 03:18:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:55.169Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"msh-tsgx12\", asset_id=\"113\", cpu=\"10\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545863391
Jul 17 03:18:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:55.190Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db249\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545858659
Jul 17 03:18:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:55.597Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_labels\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.traefik.v1\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545862058
Jul 17 03:18:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:55.619Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", container=\"configmap-reloader\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Async\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545852925
Jul 17 03:18:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:55.643Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_metadata_generation\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"cm-policy-local-cache-redis-master-69967b5879\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545862058
Jul 17 03:18:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:55.660Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_created\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-pushgateway\", uid=\"31dd9238-d4fd-4708-9137-b5324e149a22\"}" timestamp=1689545862058
Jul 17 03:18:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:55.731Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"pcap-tsgx07\", asset_id=\"85\", condition=\"true\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"cm-policy-local-cache-redis-master-69967b5879-mnc4l\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"ae4f02ae-808a-4f27-b376-d8b3a79b4ccd\"}" timestamp=1689545862058
Jul 17 03:18:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:55.745Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"40\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545859409
Jul 17 03:18:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:55.801Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth40581f6b\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545859409
Jul 17 03:18:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:56.211Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"1\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545865883
Jul 17 03:18:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:56.240Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"40\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545865883
Jul 17 03:18:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:56.263Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:20-0000:20:03.1-0000:21:00.0-infiniband_mad-umad2.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689545863372
Jul 17 03:18:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:56.280Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-rdma-devices-mlx5_6:1-umad.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689545863372
Jul 17 03:18:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:56.365Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_commands_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cmd=\"get\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545866475
Jul 17 03:18:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:56.374Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_writes_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", container=\"telegraf\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689545861260
Jul 17 03:18:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:56.440Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_rss\", asset=\"pcap-tsgx03\", asset_id=\"73\", container=\"firewall\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689545857067
Jul 17 03:18:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:56.517Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_reads_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", container=\"traefik\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545865255
Jul 17 03:18:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:56.518Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_protocol_type\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens2f0np0\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545865883
Jul 17 03:18:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:56.563Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_resident_memory_bytes\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"rdma-ndd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689545863372
Jul 17 03:18:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:56.715Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_load_average_10s\", asset=\"pcap-tsgx03\", asset_id=\"73\", container=\"metrics\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545868354
Jul 17 03:18:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:56.734Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_address_assign_type\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens3f1np1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545865883
Jul 17 03:18:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:56.757Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_info\", address=\"bc:97:e1:d3:3a:a0\", asset=\"pcap-tsgx03\", asset_id=\"73\", broadcast=\"ff:ff:ff:ff:ff:ff\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"eno33np0\", duplex=\"full\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operstate=\"up\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545865883
Jul 17 03:18:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:56.830Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_multicast_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens8f0np0\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545865883
Jul 17 03:18:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:56.846Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-pod8a18b7c9_5f34_4084_b472_fc3356545d52.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"slice\"}" timestamp=1689545863372
Jul 17 03:18:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:57.402Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_labels\", asset=\"pcap-tsgx03\", asset_id=\"73\", configmap=\"sapp-tsg-traffic-engine-numa-1\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545873389
Jul 17 03:18:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:57.420Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_io_time_weighted_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda3\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545866847
Jul 17 03:18:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:57.422Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_info\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.tsg-traffic-engine-numa-0.v2\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545873389
Jul 17 03:18:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:57.438Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"84\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545865883
Jul 17 03:18:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:59.328Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_gc_duration_seconds_count\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"OLAP-Node-Exporter-pcap-dellr740-dt001\", endpoint_id=\"251\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545877559
Jul 17 03:18:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:18:59.699Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_running\", asset=\"msh-tsgx01\", asset_id=\"51\", container=\"configmap-reloader\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"configmap-reloader-6c55bbc49f-6pg96\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"c4681a04-9a52-4802-b198-48ef7958ac80\"}" timestamp=1689545867425
Jul 17 03:19:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:00.717Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_oom_events_total\", asset=\"msh-tsgx01\", asset_id=\"51\", container=\"telegraf-security\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689545870143
Jul 17 03:19:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:00.732Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"75\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545869083
Jul 17 03:19:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:00.911Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", state=\"iowaiting\"}" timestamp=1689545873106
Jul 17 03:19:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:00.998Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_labels\", asset=\"msh-tsgx01\", asset_id=\"51\", configmap=\"prometheus-server\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545867425
Jul 17 03:19:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:01.097Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_limit_bytes\", asset=\"msh-tsgx01\", asset_id=\"51\", container=\"lb-tcp-80\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545874593
Jul 17 03:19:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:01.138Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas_available\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"tsg-traffic-engine-numa-1-firewall\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545867425
Jul 17 03:19:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:02.771Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_created\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"kubernetes-dashboard-66dd5b456\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545871412
Jul 17 03:19:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:03.022Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_timex_pps_stability_exceeded_total\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545878318
Jul 17 03:19:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:03.037Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"msh-tsgx02\", asset_id=\"63\", cpu=\"115\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545878318
Jul 17 03:19:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:03.080Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"ipmi.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"oneshot\", type=\"service\"}" timestamp=1689545868447
Jul 17 03:19:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:03.115Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_owner\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", owner_is_controller=\"true\", owner_kind=\"Job\", owner_name=\"helm-install-traefik\", pod=\"helm-install-traefik-ph7rq\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"eacc1d44-839a-43ee-bf5c-0bccf9558d70\"}" timestamp=1689545871412
Jul 17 03:19:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:03.196Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-pod87913277_0e7d_4545_8517_0068668d7130.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"slice\"}" timestamp=1689545868447
Jul 17 03:19:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:03.509Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_swap_limit_bytes\", asset=\"msh-tsgx02\", asset_id=\"63\", container=\"kube-state-metrics\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545872003
Jul 17 03:19:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:03.581Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failcnt\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689545867488
Jul 17 03:19:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:03.743Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"net_conntrack_dialer_conn_failed_total\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", dialer_name=\"snmp_exporter_metrics\", instance=\"10.10.20.159:10090\", reason=\"refused\", type=\"global\"}" timestamp=1689545882990
Jul 17 03:19:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:03.905Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"prometheus_target_metadata_cache_entries\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", instance=\"10.10.20.159:10090\", scrape_job=\"107\", type=\"global\"}" timestamp=1689545882990
Jul 17 03:19:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:04.657Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_processes\", asset=\"msh-tsgx02\", asset_id=\"63\", container=\"prometheus-alertmanager-configmap-reload\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545857704
Jul 17 03:19:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:04.672Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_data_transmitted_bytes_total\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_0\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545878318
Jul 17 03:19:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:04.733Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"prometheus_http_requests_total\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", code=\"200\", datacenter=\"MSH\", handler=\"/api/v1/metadata\", instance=\"10.10.20.159:10090\", type=\"global\"}" timestamp=1689545882990
Jul 17 03:19:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:04.744Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"prometheus_target_metadata_cache_bytes\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", instance=\"10.10.20.159:10090\", scrape_job=\"465\", type=\"global\"}" timestamp=1689545882990
Jul 17 03:19:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:04.828Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"prometheus_target_metadata_cache_entries\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", instance=\"10.10.20.159:10090\", scrape_job=\"127\", type=\"global\"}" timestamp=1689545882990
Jul 17 03:19:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:04.933Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"net_conntrack_dialer_conn_closed_total\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", dialer_name=\"471\", instance=\"10.10.20.159:10090\", type=\"global\"}" timestamp=1689545882990
Jul 17 03:19:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:04.939Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"net_conntrack_dialer_conn_established_total\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", dialer_name=\"233\", instance=\"10.10.20.159:10090\", type=\"global\"}" timestamp=1689545882990
Jul 17 03:19:05 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:05.207Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"msh-dellr740-olap005\", asset_id=\"19\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Spark-msh-dellr740-olap005\", endpoint_id=\"17\", module=\"NC-Spark\", module_id=\"47\", name=\"29\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", type=\"Processor\"}" timestamp=1689545883390
Jul 17 03:19:05 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:05.224Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_errs_total\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens3f0np0\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545878318
Jul 17 03:19:05 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:05.228Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_frequency_max_hertz\", asset=\"msh-dellr740-olap005\", asset_id=\"19\", cpu=\"27\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Spark-msh-dellr740-olap005\", endpoint_id=\"17\", module=\"NC-Spark\", module_id=\"47\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545883390
Jul 17 03:19:05 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:05.244Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"msh-tsgx02\", asset_id=\"63\", cpu=\"78\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545878318
Jul 17 03:19:05 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:05.265Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"net_conntrack_dialer_conn_closed_total\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", dialer_name=\"219\", instance=\"10.10.20.159:10090\", type=\"global\"}" timestamp=1689545882990
Jul 17 03:19:05 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:05.342Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx02\", asset_id=\"63\", container=\"coredns\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Total\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545859980
Jul 17 03:19:05 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:05.853Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_speed_bytes\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens3f1np1\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545878318
Jul 17 03:19:05 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:05.880Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Dup_Udp_Bit\", app_name=\"sapp4\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", vsysId=\"1\"}" timestamp=1689545873892
Jul 17 03:19:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:06.259Z caller=dedupe.go:112 component=remote level=info remote_name=52db6f url=http://10.10.10.159:10090/nz-agent/prometheus/proxy/api/v1/write msg="Dropped sample for series that was not explicitly dropped via relabelling" ref=4436650
Jul 17 03:19:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:07.029Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_start_time_seconds\", asset=\"twa-tsgx01\", asset_id=\"91\", container=\"telegraf-security\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689545882966
Jul 17 03:19:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:07.055Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cpu=\"19\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545883910
Jul 17 03:19:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:07.081Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cpu=\"23\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545883910
Jul 17 03:19:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:07.129Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx01\", asset_id=\"91\", container=\"dign-server-dns\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Write\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545879653
Jul 17 03:19:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:07.191Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_start_time_seconds\", asset=\"twa-tsgx01\", asset_id=\"91\", container=\"prometheus-server-configmap-reload\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545882966
Jul 17 03:19:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:07.224Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_free_bytes\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"tmpfs\", endpoint=\"NC-Spark-msh-dellr740-olap003\", endpoint_id=\"13\", fstype=\"tmpfs\", module=\"NC-Spark\", module_id=\"47\", mountpoint=\"/run/user/0\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545886091
Jul 17 03:19:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:07.234Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2dpath-pci\\\\x2d0000:81:00.0\\\\x2dscsi\\\\x2d0:2:0:0.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689545873582
Jul 17 03:19:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:07.290Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx01\", asset_id=\"91\", container=\"dign-server-web\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Write\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545870204
Jul 17 03:19:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:07.548Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx01\", asset_id=\"91\", container=\"dign-server-dns\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Async\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545879653
Jul 17 03:19:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:07.781Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_state_started\", asset=\"twa-tsgx01\", asset_id=\"91\", container=\"lb-tcp-443\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"svclb-traefik-2d88d210-xs8jw\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"a99c67a0-fa8b-483f-afda-9724525367bb\"}" timestamp=1689545881227
Jul 17 03:19:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:07.809Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_labels\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.tsg-traffic-engine-numa-0.v1\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545881227
Jul 17 03:19:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:07.837Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", fstype=\"ext4\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/var\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545883910
Jul 17 03:19:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:07.879Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_scrape_collector_duration_seconds\", asset=\"twa-tsgx01\", asset_id=\"91\", collector=\"textfile\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545883910
Jul 17 03:19:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:07.900Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cpu=\"93\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545883910
Jul 17 03:19:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:07.904Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_fifo_total\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens1f0np0\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545883910
Jul 17 03:19:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:08.019Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_down_changes_total\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"cnibr0\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545883910
Jul 17 03:19:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:08.160Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_system_seconds_total\", asset=\"twa-tsgx01\", asset_id=\"91\", container=\"prometheus-server-configmap-reload\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545879705
Jul 17 03:19:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:08.224Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_name_assign_type\", asset=\"msh-tsgx03\", asset_id=\"59\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"eno36np3\", endpoint=\"TSG-OS-msh-tsgx03\", endpoint_id=\"427\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545874311
Jul 17 03:19:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:08.325Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cpu=\"4\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545883910
Jul 17 03:19:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:08.422Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_errs_total\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens1f1np1\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545883910
Jul 17 03:19:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:08.536Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_drop_total\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth0799f3ea\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545883910
Jul 17 03:19:09 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:09.874Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_avgStoreFileAge\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"e14b6811437ae968954ef3ada3528474\", table=\"firewall_hos_bucket\", type=\"data\"}" timestamp=1689545888804
Jul 17 03:19:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:11.727Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_readRequestCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"session_record_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"cd8e1f9c9ff2a0472e77312622fcfd11\", table=\"index_partfile_session_record_hos_bucket\", type=\"index\"}" timestamp=1689545890149
Jul 17 03:19:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:11.742Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_numBytesCompactedCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"268471f048bc85ce60683d08da793a10\", table=\"assessment_hos_bucket\", type=\"data\"}" timestamp=1689545890149
Jul 17 03:19:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:11.780Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_4_metric_majorCompactionInputFileCount_99th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545890149
Jul 17 03:19:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:11.799Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_getCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", namespace=\"SYSTEM\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"af563a172b4520181206ade9e5414ef8\", table=\"LOG\", type=\"data\"}" timestamp=1689545890149
Jul 17 03:19:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:11.842Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_4_metric_compactionTime_99th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545890149
Jul 17 03:19:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:11.857Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_session_record_hos_bucket_metric_deleteTime_98th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545890149
Jul 17 03:19:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:11.858Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_3_metric_majorCompactionInputSize_98th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545890149
Jul 17 03:19:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:11.882Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_numReferenceFiles\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"rtp_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"afda67220314ac681ee8f19c890bcc36\", table=\"index_partfile_rtp_hos_bucket\", type=\"index\"}" timestamp=1689545890149
Jul 17 03:19:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:11.897Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_194_4_metric_scanSize_95th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545890149
Jul 17 03:19:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:11.913Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_memStoreSize\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"00dc5a251addf7b4aaed51561226ada8\", table=\"proxy_hos_bucket\", type=\"data\"}" timestamp=1689545890149
Jul 17 03:19:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:12.154Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_compactionsCompletedCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"58414b9635e704bfdbebc3459bee598f\", table=\"assessment_hos_bucket\", type=\"data\"}" timestamp=1689545890149
Jul 17 03:19:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:15.245Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"74\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545881230
Jul 17 03:19:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:15.264Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_drop_total\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethe08168cb\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545881230
Jul 17 03:19:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:15.272Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_packets_transmitted_total\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_11\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545881230
Jul 17 03:19:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:15.282Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-e53087b8c81225f82a3933033908076420ba8996b9d2775983ccd3d9a9fdbbc5.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"scope\"}" timestamp=1689545887046
Jul 17 03:19:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:15.290Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"twa-tsgx07\", asset_id=\"103\", condition=\"Available\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"kubernetes-dashboard\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"true\"}" timestamp=1689545888076
Jul 17 03:19:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:15.303Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:e0-0000:e0:03.1-0000:e2:00.0-infiniband_mad-umad10.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689545887046
Jul 17 03:19:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:15.318Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_colls_total\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"br_dign_s\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545881230
Jul 17 03:19:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:15.326Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_running\", asset=\"twa-tsgx07\", asset_id=\"103\", container=\"prometheus-pushgateway\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-pushgateway-bd476bcf9-d7fgf\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"eda00de6-45a4-4eaf-8b0e-75c0b9c1b061\"}" timestamp=1689545888076
Jul 17 03:19:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:15.354Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_compressed_total\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth6fd4be6c\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545881230
Jul 17 03:19:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:15.415Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_resident_memory_bytes\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-journald.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689545887046
Jul 17 03:19:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:15.422Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_start_time_seconds\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689545889150
Jul 17 03:19:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:15.432Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-a279e453\\\\x2d6b3f\\\\x2d4a02\\\\x2dbbf2\\\\x2dbcc1e97e842b-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2d4wsx4.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689545887046
Jul 17 03:19:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:15.470Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"twa-tsgx07\", asset_id=\"103\", condition=\"false\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-alertmanager-6df7f457b6-nwt7p\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"a8ca687f-46d5-4647-ba3b-841b55595ad2\"}" timestamp=1689545888076
Jul 17 03:19:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:15.665Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_running\", asset=\"twa-tsgx07\", asset_id=\"103\", container=\"telegraf\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", pod=\"tsg-traffic-engine-numa-1-firewall-574b9bd44b-zm8zv\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"704b5479-b3a5-4f40-a111-190391574a21\"}" timestamp=1689545888076
Jul 17 03:19:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:15.675Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_namespace_status_phase\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", phase=\"Terminating\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545888076
Jul 17 03:19:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:15.688Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_type\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.tsg-traffic-engine-numa-0.v5\", service=\"prometheus-kube-state-metrics\", type=\"helm.sh/release.v1\"}" timestamp=1689545888076
Jul 17 03:19:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:15.789Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-sda.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689545887046
Jul 17 03:19:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:15.798Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx07\", asset_id=\"103\", container=\"dign-client\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Write\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545883079
Jul 17 03:19:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:15.799Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_id\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth6994e71f\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545881230
Jul 17 03:19:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:15.843Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kdump.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"service\"}" timestamp=1689545887046
Jul 17 03:19:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:15.876Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"polkit.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"service\"}" timestamp=1689545887046
Jul 17 03:19:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:16.955Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_commands_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cmd=\"incrby\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545886027
Jul 17 03:19:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:20.034Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"17\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545897612
Jul 17 03:19:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:20.251Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"44\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545897612
Jul 17 03:19:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:20.302Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"33\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545888263
Jul 17 03:19:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:20.508Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"85\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545897612
Jul 17 03:19:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:20.566Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_info\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_endpoint=\"ipmi-exporter\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545893828
Jul 17 03:19:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:20.585Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_node_status_allocatable\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", node=\"localhost\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", resource=\"memory\", service=\"prometheus-kube-state-metrics\", unit=\"byte\"}" timestamp=1689545893828
Jul 17 03:19:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:20.594Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db15\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545894840
Jul 17 03:19:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:20.627Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"NetworkManager-wait-online.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"service\"}" timestamp=1689545890657
Jul 17 03:19:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:20.645Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_ready\", asset=\"pcap-tsgx10\", asset_id=\"89\", container=\"helm\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-mrzcpd-exporter-wvc54\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"f3514640-3d29-4631-9fb4-2435e27890f8\"}" timestamp=1689545893828
Jul 17 03:19:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:20.762Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_max_usage_bytes\", asset=\"twa-tsgx06\", asset_id=\"101\", container=\"ipmi-exporter\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545883094
Jul 17 03:19:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:20.781Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_metadata_resource_version\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.tsg-traffic-engine-numa-1.v2\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545893949
Jul 17 03:19:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:20.782Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"shm\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/89e2d284072c918459494a673de1758c0e35cfa8d573f12839747dd5eb98d0a1/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545888263
Jul 17 03:19:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:20.810Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_waiting\", asset=\"pcap-tsgx10\", asset_id=\"89\", container=\"helm\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-prometheus-fmhhw\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"4f84a780-549d-4f40-bdec-aee8906d8427\"}" timestamp=1689545893828
Jul 17 03:19:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:20.830Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth0d28b4f8\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545888263
Jul 17 03:19:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:20.844Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_reads_total\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/sys/fs/cgroup\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545887769
Jul 17 03:19:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:20.846Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_bytes_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens1f1np1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545888263
Jul 17 03:19:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:20.892Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"twa-tsgx06\", asset_id=\"101\", container=\"dign-client\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"cadvisor\", failure_type=\"pgmajfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"container\"}" timestamp=1689545889459
Jul 17 03:19:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:20.980Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-38e3a560b644b18ec47fddbe282b76d0f18f92096cb0d76cc9e55d5a2b9dad2b.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"scope\"}" timestamp=1689545890657
Jul 17 03:19:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:21.024Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Eth_Pkt\", app_name=\"sapp4\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", vsysId=\"1\"}" timestamp=1689545899077
Jul 17 03:19:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:21.039Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", container=\"prometheus-alertmanager\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"cadvisor\", failure_type=\"pgmajfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"container\"}" timestamp=1689545892192
Jul 17 03:19:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:21.065Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_load_average_10s\", asset=\"pcap-tsgx10\", asset_id=\"89\", container=\"exporter\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545884409
Jul 17 03:19:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:21.203Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", container=\"configmap-reloader\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Read\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545884097
Jul 17 03:19:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:21.344Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_packets_dropped_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"cadvisor\", interface=\"br_dign_s\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545894783
Jul 17 03:19:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:21.361Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_type\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"chart-values-tsg-traffic-engine-numa-1\", service=\"prometheus-kube-state-metrics\", type=\"Opaque\"}" timestamp=1689545893828
Jul 17 03:19:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:21.442Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", phase=\"Pending\", pod=\"configmap-reloader-6c55bbc49f-sdx4c\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"d69b2139-72e0-47ab-b29f-fa787b9e63b9\"}" timestamp=1689545893828
Jul 17 03:19:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:21.524Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"1\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545888263
Jul 17 03:19:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:21.595Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_start_time_seconds\", asset=\"pcap-tsgx10\", asset_id=\"89\", container=\"prometheus-alertmanager\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545895735
Jul 17 03:19:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:21.780Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_reads_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", container=\"dign-server-web\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545877974
Jul 17 03:19:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:23.630Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx09\", asset_id=\"55\", cpu=\"11\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545894790
Jul 17 03:19:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:23.925Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_queue_length\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"vethcfb0293a\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545894790
Jul 17 03:19:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:23.994Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_bytes_total\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens3f0np0\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545894790
Jul 17 03:19:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:24.036Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db46\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545897019
Jul 17 03:19:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:24.078Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-e29ba7f3c6e483ee504e55d39207f8bc66701a3ab2cd81ede609c6ee5b1d703d.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"scope\"}" timestamp=1689545898416
Jul 17 03:19:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:24.090Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"system-systemd\\\\x2dfsck.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"slice\"}" timestamp=1689545898416
Jul 17 03:19:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:24.198Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Running\", pod=\"svclb-traefik-53c779fd-wwnxx\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"17a39b41-f716-4eab-a979-0f0f67f99785\"}" timestamp=1689545900068
Jul 17 03:19:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:24.303Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_oom_events_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"metrics-server\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545896713
Jul 17 03:19:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:24.336Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_storageclass_info\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", provisioner=\"rancher.io/local-path\", reclaim_policy=\"Delete\", service=\"prometheus-kube-state-metrics\", storageclass=\"local-path\", volume_binding_mode=\"WaitForFirstConsumer\"}" timestamp=1689545900068
Jul 17 03:19:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:24.397Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"15\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545894790
Jul 17 03:19:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:24.401Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db165\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545897019
Jul 17 03:19:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:24.426Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_metadata_resource_version\", asset=\"pcap-tsgx12\", asset_id=\"117\", configmap=\"multus-cni-config\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545891122
Jul 17 03:19:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:24.443Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"traefik-64f55bb67d-xmdbm\", project=\"TSG-OS\", project_id=\"7\", reason=\"UnexpectedAdmissionError\", service=\"prometheus-kube-state-metrics\", uid=\"012398ff-fe58-4ee4-8731-8e61bf317bbf\"}" timestamp=1689545900068
Jul 17 03:19:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:24.447Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"36\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545900160
Jul 17 03:19:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:24.459Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"7\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545900160
Jul 17 03:19:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:24.477Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_files_free\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"shm\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/2cecd16a141d9d68b235173be7c3e8115692fc6e05b6fdb1d80145067687e261/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545900160
Jul 17 03:19:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:24.487Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_observed_generation\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"configmap-reloader\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545900068
Jul 17 03:19:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:24.492Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db95\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545901795
Jul 17 03:19:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:24.499Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_sockstat_UDP_inuse\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545900160
Jul 17 03:19:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:24.525Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_commands_failed_calls_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cmd=\"incrby\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545901795
Jul 17 03:19:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:24.542Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx09\", asset_id=\"55\", cpu=\"112\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545894790
Jul 17 03:19:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:24.555Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx09\", asset_id=\"55\", cpu=\"95\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545894790
Jul 17 03:19:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:24.618Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_daemonset_status_updated_number_scheduled\", asset=\"msh-tsgx09\", asset_id=\"55\", daemonset=\"dign-client\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545900068
Jul 17 03:19:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:24.677Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"dashboard-metrics-scraper\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Discard\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545897572
Jul 17 03:19:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:24.689Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_threads\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"mrapm-device\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545893377
Jul 17 03:19:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:24.880Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_compressed_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"vethfc5d9f9d\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545900160
Jul 17 03:19:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:26.064Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_oom_events_total\", asset=\"twa-tsgx09\", asset_id=\"107\", container=\"coredns\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545902994
Jul 17 03:19:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:26.118Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"phy_rx_error_total\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", dev=\"ens1f1np1\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"mrzcpd-exporter\"}" timestamp=1689545898295
Jul 17 03:19:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:26.232Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545895810
Jul 17 03:19:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:26.251Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"clickhouse_osio_wait_time_cpu47\", asset=\"twa-dellr740-dt002\", asset_id=\"39\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"NC-Clickhouse-twa-dellr740-dt002\", endpoint_id=\"89\", module=\"NC-Clickhouse\", module_id=\"29\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545904866
Jul 17 03:19:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:26.269Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db49\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545903549
Jul 17 03:19:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:26.286Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_changes_total\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"virtio_dign_c\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545895810
Jul 17 03:19:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:26.388Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"65\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545895810
Jul 17 03:19:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:26.650Z caller=dedupe.go:112 component=remote level=error remote_name=52db6f url=http://10.10.10.159:10090/nz-agent/prometheus/proxy/api/v1/write msg="non-recoverable error" count=1000 exemplarCount=0 err="server returned HTTP status 400 : out of order sample"
Jul 17 03:19:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:26.712Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-rdma-devices-mlx5_0:1-umad.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689545893907
Jul 17 03:19:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:26.806Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"system-systemd\\\\x2dfsck.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"slice\"}" timestamp=1689545893907
Jul 17 03:19:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:27.021Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"42\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545895810
Jul 17 03:19:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:27.107Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"twa-tsgx09\", asset_id=\"107\", container=\"mrapm-device\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"sleeping\"}" timestamp=1689545899668
Jul 17 03:19:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:27.141Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_drop_total\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"lo\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545895810
Jul 17 03:19:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:27.227Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"twa-tsgx09\", asset_id=\"107\", container=\"metrics\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"uninterruptible\"}" timestamp=1689545895631
Jul 17 03:19:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:27.232Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_address_assign_type\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth45cfb0fe\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545895810
Jul 17 03:19:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:27.268Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_info\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", job_name=\"helm-install-traefik-crd\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545898744
Jul 17 03:19:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:27.406Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"98\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545902882
Jul 17 03:19:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:27.432Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"twa-tsgx09\", asset_id=\"107\", container=\"redis\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"uninterruptible\"}" timestamp=1689545890090
Jul 17 03:19:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:27.439Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"119\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545895810
Jul 17 03:19:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:27.446Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-c5e2255d034116d68d43cdcc2ab961d5bc3665041ec119fe6fddb48920f0571b-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689545894114
Jul 17 03:19:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:27.453Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"tmpfs\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/var/lib/kubelet/pods/1f68b554-5e45-4a8c-8e86-fb16303ebab5/volumes/kubernetes.io~projected/kube-api-access-hs4n5\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545902882
Jul 17 03:19:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:27.477Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_last_seen\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689545904865
Jul 17 03:19:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:27.494Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"21\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545895810
Jul 17 03:19:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:27.509Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"cm-policy-local-cache-redis-master-69967b5879-fpcq2\", project=\"TSG-OS\", project_id=\"7\", reason=\"NodeLost\", service=\"prometheus-kube-state-metrics\", uid=\"78ab1f97-9ecb-47ed-ae81-048055fb4556\"}" timestamp=1689545895682
Jul 17 03:19:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:27.616Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"twa-tsgx09\", asset_id=\"107\", condition=\"false\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-node-exporter-h5kns\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"151c8cb0-3185-4fb1-bbff-a389c3a473df\"}" timestamp=1689545898744
Jul 17 03:19:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:27.632Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545895810
Jul 17 03:19:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:27.648Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"123\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545895810
Jul 17 03:19:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:27.657Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-e608d5789ad55db57cde72ce26e5d64eb172ad8dd0caa03d940aadf5e68899c3-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689545893907
Jul 17 03:19:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:27.674Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_compressed_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth2e13d535\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545902882
Jul 17 03:19:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:27.689Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"28\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545902882
Jul 17 03:19:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:27.761Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_packets_dropped_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", interface=\"eno34np1\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545898627
Jul 17 03:19:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:27.796Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"73\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545895810
Jul 17 03:19:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:28.018Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-46b1016fe0882f640c1330cdff069a4aa4775b598d30e8571ff0c78026146d00-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689545894114
Jul 17 03:19:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:28.068Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"phy_tx_packets_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", dev=\"ens1f0np0\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"mrzcpd-exporter\"}" timestamp=1689545893863
Jul 17 03:19:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:28.116Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"34\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545902882
Jul 17 03:19:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:28.335Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", pod=\"kubernetes-dashboard-66dd5b456-2cjmj\", project=\"TSG-OS\", project_id=\"7\", reason=\"UnexpectedAdmissionError\", service=\"prometheus-kube-state-metrics\", uid=\"fa19b2ec-5070-431e-b30c-dc1e010c4b86\"}" timestamp=1689545895682
Jul 17 03:19:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:28.378Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_netstat_Udp6_SndbufErrors\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545902882
Jul 17 03:19:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:28.589Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-0b0292d1\\\\x2d7c6a\\\\x2d4fad\\\\x2da1f9\\\\x2df14e3c7cb4f8-volume\\\\x2dsubpaths-config\\\\x2dvolume-firewall-1.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689545894114
Jul 17 03:19:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:28.642Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"4\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545902882
Jul 17 03:19:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:28.648Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"loki_logql_querystats_chunk_download_latency_seconds_bucket\", agent_name=\"msh-kvm-nz001\", application=\"loki\", datacenter=\"MSH\", instance=\"127.0.0.1:13100\", le=\"64.0\", range=\"instant\", status_code=\"500\", type=\"metric\"}" timestamp=1689545908327
Jul 17 03:19:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:28.655Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_compressed_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"vethefb0a7e8\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545902882
Jul 17 03:19:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:28.663Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"loki_response_message_bytes_count\", agent_name=\"msh-kvm-nz001\", application=\"loki\", datacenter=\"MSH\", instance=\"127.0.0.1:13100\", method=\"gRPC\", route=\"/logproto.Pusher/Push\"}" timestamp=1689545908327
Jul 17 03:19:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:28.702Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"cortex_kv_request_duration_seconds_bucket\", agent_name=\"msh-kvm-nz001\", application=\"loki\", datacenter=\"MSH\", instance=\"127.0.0.1:13100\", kv_name=\"distributor-lifecycler\", le=\"0.05\", operation=\"CAS\", role=\"primary\", status_code=\"200\", type=\"memberlist\"}" timestamp=1689545908327
Jul 17 03:19:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:28.764Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db38\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545899446
Jul 17 03:19:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:31.161Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"31\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545900939
Jul 17 03:19:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:31.399Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"115\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545900939
Jul 17 03:19:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:31.479Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-sda5.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689545901015
Jul 17 03:19:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:31.483Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-user-sessions.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"service\"}" timestamp=1689545901015
Jul 17 03:19:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:31.705Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_created\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"cm-cache-extra-secrets\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545910436
Jul 17 03:19:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:31.732Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"76\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545899093
Jul 17 03:19:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:31.745Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_hwmon_chip_names\", asset=\"twa-tsgx03\", asset_id=\"95\", chip=\"0000:60:03_1_0000:63:00_2\", chip_name=\"bnxt_en\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545899093
Jul 17 03:19:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:31.767Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_id\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth1618597b\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545899093
Jul 17 03:19:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:31.785Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"cadvisor_version_info\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", kernelVersion=\"5.17.15-1.el8.x86_64\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", osVersion=\"TSG-OS v23.07-rc1-079e964 (TSGXNXR620G40R01P0906)\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545902009
Jul 17 03:19:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:31.795Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_load_average_10s\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545895396
Jul 17 03:19:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:31.799Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"92\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545899093
Jul 17 03:19:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:31.816Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_netstat_Ip6_InOctets\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545899093
Jul 17 03:19:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:31.851Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"msh-tsgx05\", asset_id=\"67\", condition=\"false\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-delete-tsg-traffic-engine-numa-1-hxpr6\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"c86bafef-0bfe-4d1f-bcc1-adc8b0a94ad0\"}" timestamp=1689545904068
Jul 17 03:19:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:31.857Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_status_succeeded\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", job_name=\"helm-install-tsg-traffic-engine-numa-1\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545910436
Jul 17 03:19:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:31.893Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_labels\", asset=\"msh-tsgx05\", asset_id=\"67\", configmap=\"chart-content-traefik\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545904068
Jul 17 03:19:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:31.942Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"etc.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689545901015
Jul 17 03:19:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:31.945Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_reads_total\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/sys/fs/cgroup\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545901728
Jul 17 03:19:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:31.961Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"firewall\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", failure_type=\"pgmajfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"container\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689545896522
Jul 17 03:19:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:32.165Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"59\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545899093
Jul 17 03:19:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:32.194Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"60\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545899093
Jul 17 03:19:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:32.288Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_io_current\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545901728
Jul 17 03:19:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:32.433Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-firstboot.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"service\"}" timestamp=1689545906907
Jul 17 03:19:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:32.582Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"66\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545900791
Jul 17 03:19:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:32.625Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"90\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545900791
Jul 17 03:19:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:32.655Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"50\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545899093
Jul 17 03:19:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:32.662Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"81\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545900791
Jul 17 03:19:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:32.671Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_errs_total\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens3f1np1\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545900791
Jul 17 03:19:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:32.724Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Read\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545901785
Jul 17 03:19:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:32.742Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"26\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545899093
Jul 17 03:19:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:32.863Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"98\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545900791
Jul 17 03:19:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:32.895Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"47\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545900791
Jul 17 03:19:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:32.922Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_virtual_memory_max_bytes\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"watchdog.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689545907691
Jul 17 03:19:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:32.965Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-tmpfiles-setup.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689545907691
Jul 17 03:19:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:32.990Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"basic.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"target\"}" timestamp=1689545907691
Jul 17 03:19:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:33.002Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dbus.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"service\"}" timestamp=1689545902644
Jul 17 03:19:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:33.361Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_max_fds\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"rsyslog.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689545907691
Jul 17 03:19:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:33.374Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-e02c929a\\\\x2d3c07\\\\x2d4477\\\\x2db04a\\\\x2d8ffc16c405f5-volumes-kubernetes.io\\\\x7esecret-kubernetes\\\\x2ddashboard\\\\x2dcerts.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689545907691
Jul 17 03:19:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:33.377Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_file_descriptors\", asset=\"twa-tsgx05\", asset_id=\"99\", container=\"monit-stream\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545896641
Jul 17 03:19:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:33.381Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_info\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_endpoint=\"prometheus-kube-state-metrics\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545899276
Jul 17 03:19:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:33.438Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"111\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545900605
Jul 17 03:19:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:33.455Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-a3c48cd45eea1a45390f23161076ae0f439f3130749e4b1ca84871cccd9ff471.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"scope\"}" timestamp=1689545907691
Jul 17 03:19:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:33.484Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-9982afd0\\\\x2d5736\\\\x2d4dce\\\\x2db764\\\\x2d2f7b0c30831a-volume\\\\x2dsubpaths-config\\\\x2dvolume-firewall-1.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689545907691
Jul 17 03:19:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:33.492Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"tsg_master_lpi\", app_name=\"tsg_master\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"traffic-engine\", line_name=\"sum\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", table_name=\"sync_application\", vsysId=\"1\"}" timestamp=1689545903815
Jul 17 03:19:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:33.817Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"32\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545900605
Jul 17 03:19:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:33.840Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_annotations\", asset=\"twa-tsgx05\", asset_id=\"99\", configmap=\"cm-policy-local-cache-redis-health\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545899894
Jul 17 03:19:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:33.858Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_data_received_bytes_total\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_9\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545900605
Jul 17 03:19:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:33.859Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_restarts_total\", asset=\"twa-tsgx05\", asset_id=\"99\", container=\"configmap-reloader\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"configmap-reloader-6c55bbc49f-tvvkl\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"5eafbc6c-7214-4748-a907-76b294823901\"}" timestamp=1689545899894
Jul 17 03:19:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:33.875Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_bytes_total\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth50f7b1ca\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545900605
Jul 17 03:19:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:33.914Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_created\", asset=\"twa-tsgx05\", asset_id=\"99\", configmap=\"telegraf-tsg-traffic-engine-numa-0\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545899894
Jul 17 03:19:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:34.082Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"119\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545900605
Jul 17 03:19:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:40.465Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_arp_entries\", asset=\"twa-dellr740-dt003\", asset_id=\"41\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"em1\", endpoint=\"OLAP-Node-Exporter-twa-dellr740-dt003\", endpoint_id=\"261\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"hbase_master,clickhouse_data,kafka,gohangout,zookeeper,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545918473
Jul 17 03:19:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:40.676Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_drop_total\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens8f0np0\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545911647
Jul 17 03:19:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:40.755Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth4e7244c8\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545911647
Jul 17 03:19:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:40.796Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_hbase_table_rsgroup_metric_splitTime_90th_percentile\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545919756
Jul 17 03:19:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:40.871Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_appendCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"default\", project=\"OLAP\", project_id=\"3\", region=\"d839716bb431f8ae37418c3ce87ab845\", table=\"index_partfile_default\", type=\"index\"}" timestamp=1689545919756
Jul 17 03:19:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:40.884Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_hbase_table_rsgroup_metric_splitTime_25th_percentile\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545919756
Jul 17 03:19:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:40.900Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_memory_max_bytes\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545913827
Jul 17 03:19:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:40.912Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-7fed6579\\\\x2d2b87\\\\x2d448c\\\\x2d8a58\\\\x2dd522c3843522-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2dznpzw.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689545915764
Jul 17 03:19:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:41.000Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"122\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545911647
Jul 17 03:19:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:41.019Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"76\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545911647
Jul 17 03:19:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:41.039Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_hbase_table_meta_metric_scanSize_99th_percentile\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545919756
Jul 17 03:19:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:41.076Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_putCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"firewall_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"6b0ca914ac067acc26419d3a7e6877ce\", table=\"index_filename_firewall_hos_bucket\", type=\"index\"}" timestamp=1689545919756
Jul 17 03:19:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:41.112Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_appendCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"2810ac74467d2470e18fe27ebbda6b41\", table=\"firewall_hos_bucket\", type=\"data\"}" timestamp=1689545919756
Jul 17 03:19:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:41.146Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"6\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545911647
Jul 17 03:19:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:41.153Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"13\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545911647
Jul 17 03:19:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:41.167Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"2\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545911647
Jul 17 03:19:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:41.286Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_10_194_1_metric_incrementTime_90th_percentile\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545919756
Jul 17 03:19:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:41.348Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_storeFileSize\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"assessment_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"27576aeb33671b88e7a5aa4de5015e12\", table=\"index_filename_assessment_hos_bucket\", type=\"index\"}" timestamp=1689545919756
Jul 17 03:19:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:41.515Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_10_193_1_metric_maxStoreFileAge\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545919756
Jul 17 03:19:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:41.640Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_system_table_bucket_meta_metric_majorCompactionOutputSize_90th_percentile\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545919756
Jul 17 03:19:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:41.810Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_hbase_table_rsgroup_metric_compactionInputSize_num_ops\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545919756
Jul 17 03:19:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:42.035Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"121\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545911647
Jul 17 03:19:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:42.208Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_cpu_seconds_total\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-pod2133adaf_cc90_44bb_9cd8_e180b9c1e90b.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"slice\"}" timestamp=1689545915764
Jul 17 03:19:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:42.277Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_ScanTime_99th_percentile\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Server\"}" timestamp=1689545919756
Jul 17 03:19:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:42.324Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_10_194_1_metric_majorCompactionInputSize_min\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545919756
Jul 17 03:19:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:42.526Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-pod98daa3ea_45f0_41c2_be94_512fb3e2a016.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"slice\"}" timestamp=1689545920671
Jul 17 03:19:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:42.540Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_LOG_metric_appendTime_median\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545919756
Jul 17 03:19:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:42.724Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_10_194_1_metric_flushTime_95th_percentile\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545919756
Jul 17 03:19:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:42.775Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_gc_duration_seconds\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Gohangout-msh-dellr740-dt002\", endpoint_id=\"167\", module=\"DC-Gohangout\", module_id=\"35\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", quantile=\"0\"}" timestamp=1689545922623
Jul 17 03:19:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:42.777Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_minStoreFileAge\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"c0d08dd9c8090a5d8442f6cb602de9da\", table=\"session_record_hos_bucket\", type=\"data\"}" timestamp=1689545919756
Jul 17 03:19:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:43.034Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_files_free\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"shm\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/88dda23c886705dc21e9a9617bb137282356841446cd5b017bc7233624dbc807/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545916242
Jul 17 03:19:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:43.056Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_compressed_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth9134a8a9\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545916242
Jul 17 03:19:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:43.080Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"pcap-tsgx01\", asset_id=\"71\", condition=\"unknown\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-traefik-6b6lg\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"5f8aea5a-3386-417f-afa8-3bbb60c77409\"}" timestamp=1689545915154
Jul 17 03:19:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:43.141Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"3\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545916242
Jul 17 03:19:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:43.159Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_speed_bytes\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth94352f5e\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545916242
Jul 17 03:19:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:43.248Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.grpc.v1.cri-sandboxes-7fd39b22aeb9c7c3c756d0cfc15763f1c2e8d87c33e18a1b2c854c5ccd92f347-shm.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689545920671
Jul 17 03:19:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:43.256Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_up_changes_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"virtio_dign_c\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545916242
Jul 17 03:19:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:43.270Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_frame_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"lo\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545916242
Jul 17 03:19:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:43.282Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"blk-availability.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689545920671
Jul 17 03:19:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:43.367Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"mysql_global_status_performance_schema_lost_total\", asset=\"msh-dellr740-olap002\", asset_id=\"13\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Mariadb-msh-dellr740-olap002\", endpoint_id=\"69\", instrumentation=\"index_stat_lost\", module=\"NC-Mariadb\", module_id=\"43\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545922786
Jul 17 03:19:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:43.428Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_multicast_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"vethe98b5122\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545916242
Jul 17 03:19:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:43.433Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"ext4\", name=\"home.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689545920671
Jul 17 03:19:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:43.643Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", container=\"kube-state-metrics\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"cadvisor\", failure_type=\"pgmajfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"hierarchy\"}" timestamp=1689545910001
Jul 17 03:19:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:43.658Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_errs_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"eno36np3\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545916242
Jul 17 03:19:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:43.823Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_unicast_packets_transmitted_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"mlx5_11\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545916242
Jul 17 03:19:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:43.863Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"network-pre.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"target\"}" timestamp=1689545920671
Jul 17 03:19:48 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:48.661Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_2_metric_compactionTime_mean\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545927438
Jul 17 03:19:48 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:48.677Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_1_metric_incrementTime_max\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545927438
Jul 17 03:19:48 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:48.693Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_1_metric_putTime_num_ops\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545927438
Jul 17 03:19:48 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:48.716Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_1_metric_compactionOutputSize_98th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545927438
Jul 17 03:19:48 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:48.842Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_4_metric_flushOutputSize_90th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545927438
Jul 17 03:19:48 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:48.873Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_1_metric_putTime_99th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545927438
Jul 17 03:19:48 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:48.907Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_2_metric_flushedOutputBytes\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545927438
Jul 17 03:19:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:49.181Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_PostOpenTime_25th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"PhoenixIndexer\"}" timestamp=1689545927438
Jul 17 03:19:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:49.198Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_1_metric_compactionInputSize_25th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545927438
Jul 17 03:19:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:49.707Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-systemd-exporter-vq7tn\", project=\"TSG-OS\", project_id=\"7\", reason=\"Shutdown\", service=\"prometheus-kube-state-metrics\", uid=\"8760a158-2fa4-4332-99b5-4233fdf937ac\"}" timestamp=1689545919668
Jul 17 03:19:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:49.731Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_type\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.configmap-reloader.v1\", service=\"prometheus-kube-state-metrics\", type=\"helm.sh/release.v1\"}" timestamp=1689545919668
Jul 17 03:19:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:49.743Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_readRequestCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"proxy_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"e685a78146b4beeb8c55625a39a5a0f7\", table=\"index_partfile_proxy_hos_bucket\", type=\"index\"}" timestamp=1689545927438
Jul 17 03:19:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:49.764Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_compactionsFailedCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"firewall_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"e70cd5f65f1308a33afb54386df16002\", table=\"index_partfile_firewall_hos_bucket\", type=\"index\"}" timestamp=1689545927438
Jul 17 03:19:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:49.770Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_maxStoreFileAge\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"proxy_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"a3b7319ec286102d93f51022aa81c973\", table=\"index_partfile_proxy_hos_bucket\", type=\"index\"}" timestamp=1689545927438
Jul 17 03:19:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:49.779Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_194_4_metric_majorCompactionInputFileCount_99th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545927438
Jul 17 03:19:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:49.814Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_4_metric_flushOutputSize_99_9th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545927438
Jul 17 03:19:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:49.822Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_restarts_total\", asset=\"twa-tsgx08\", asset_id=\"105\", container=\"ipmi-exporter\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"ipmi-exporter-6f8464dd94-96ht4\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"02feffa3-0f34-4a13-8c7e-cf66f7d181c4\"}" timestamp=1689545921568
Jul 17 03:19:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:49.826Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_packets_total\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethe7bf01ed\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545921797
Jul 17 03:19:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:49.830Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_minStoreFileAge\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"troubleshooting_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"b301f632e0b40d5f825252a4bde96f7d\", table=\"index_filename_troubleshooting_hos_bucket\", type=\"index\"}" timestamp=1689545927438
Jul 17 03:19:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:49.831Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"nsfs\", name=\"run-netns-cni\\\\x2df4e286d6\\\\x2dee9b\\\\x2dccf9\\\\x2dda0d\\\\x2d57e1c8aa6c8c.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689545926554
Jul 17 03:19:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:49.843Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_dormant\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens2f1np1\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545921797
Jul 17 03:19:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:49.918Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_tsg_galaxy_table_relation_user_teid_metric_scanTime_90th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545927438
Jul 17 03:19:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:49.933Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-a43b9675\\\\x2dfdba\\\\x2d4957\\\\x2dab2b\\\\x2d41d084fe86c7-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2dm8jml.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689545926554
Jul 17 03:19:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:49.999Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_exceptions_OutOfOrderScannerNextException\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"IPC\"}" timestamp=1689545927438
Jul 17 03:19:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:50.003Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_getCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"default\", project=\"OLAP\", project_id=\"3\", region=\"1dc35781b98addf9940b9004125b0307\", table=\"index_filename_default\", type=\"index\"}" timestamp=1689545927438
Jul 17 03:19:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:50.042Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_lastMajorCompactionAge\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"knowledge_base_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"2528876479befa7c9103f990de0cc936\", table=\"index_time_knowledge_base_hos_bucket\", type=\"index\"}" timestamp=1689545927438
Jul 17 03:19:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:50.068Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_flushesQueuedCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"22a2444c5578382f2c9b2238d198336b\", table=\"default\", type=\"data\"}" timestamp=1689545927438
Jul 17 03:19:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:50.080Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_address_assign_type\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"cnibr0\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545926709
Jul 17 03:19:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:50.100Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_storeFileSize\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"knowledge_base_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"5d3e08ed2360c3cc0a818ebc8b8846b2\", table=\"index_time_knowledge_base_hos_bucket\", type=\"index\"}" timestamp=1689545927438
Jul 17 03:19:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:50.106Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_193_1_metric_putBatchTime_98th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545927438
Jul 17 03:19:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:50.134Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_4_metric_majorCompactionOutputSize_99_9th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545927438
Jul 17 03:19:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:50.151Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_193_1_metric_majorCompactionOutputSize_95th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545927438
Jul 17 03:19:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:50.155Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_core_throttles_total\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", core=\"4\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-olap003\", endpoint_id=\"235\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"yarn_resourcemanager,zookeeper,nacos,kafka,spark,druid_data,yarn_nodemanager\", package=\"1\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545929713
Jul 17 03:19:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:50.215Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_hbase_table_rsgroup_metric_deleteBatchTime_max\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545927438
Jul 17 03:19:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:50.223Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_write_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/data/var/lib/kubelet/pods/59e0fcfb-c5b9-4bb0-aefc-a15fed75746e/volumes/kubernetes.io~projected/kube-api-access-2wnf5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545918663
Jul 17 03:19:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:50.236Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"94\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545926709
Jul 17 03:19:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:50.240Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_ulimits_soft\", asset=\"twa-tsgx08\", asset_id=\"105\", container=\"lb-tcp-80\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", ulimit=\"max_open_files\"}" timestamp=1689545915518
Jul 17 03:19:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:50.259Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_194_2_metric_deleteBatchTime_median\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545927438
Jul 17 03:19:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:50.275Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_appendCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"troubleshooting_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"4e4a10d75a0bbcfbfd0bc2ef1602f3bf\", table=\"index_time_troubleshooting_hos_bucket\", type=\"index\"}" timestamp=1689545927438
Jul 17 03:19:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:50.312Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"import-state.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"service\"}" timestamp=1689545926554
Jul 17 03:19:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:50.437Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_dormant\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"cnibr0\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545926709
Jul 17 03:19:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:50.446Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_annotations\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"configmap-reloader\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545919668
Jul 17 03:19:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:50.449Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db58\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545926625
Jul 17 03:19:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:50.458Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"71\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545926709
Jul 17 03:19:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:50.465Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-706d1113aac372b7fbb3191e0fff56a0981e18d8acf3274c04e13a2bb1ec50d1.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"scope\"}" timestamp=1689545921057
Jul 17 03:19:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:50.504Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"73\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545921797
Jul 17 03:19:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:50.515Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_running\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"prometheus-pushgateway\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-pushgateway-bd476bcf9-8bfzq\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"c11c351e-f86b-4fa5-8167-cadbe729e2c9\"}" timestamp=1689545919668
Jul 17 03:19:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:50.522Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-rdma-devices-mlx5_5:1-umad.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689545926554
Jul 17 03:19:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:50.564Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Pkt_Outbound\", app_name=\"sapp4\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", vsysId=\"1\"}" timestamp=1689545918548
Jul 17 03:19:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:50.567Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"prometheus-server\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"iowaiting\"}" timestamp=1689545913788
Jul 17 03:19:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:50.582Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_labels\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-alertmanager-6df7f457b6-vgl6l\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"9fc73d80-3d8a-43be-8ada-97bd28ed31fd\"}" timestamp=1689545919668
Jul 17 03:19:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:50.595Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"twa-tsgx10\", asset_id=\"109\", cpu=\"115\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545921797
Jul 17 03:19:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:50.738Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-f4199c99b6b713260f774913b51c3c5fd4ec8d5d8cc8d7b11f17282194b40459-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689545926554
Jul 17 03:19:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:50.806Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx10\", asset_id=\"109\", cpu=\"83\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545921797
Jul 17 03:19:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:50.836Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_fifo_total\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens8f0np0\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545921797
Jul 17 03:19:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:50.836Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_queue_length\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens3f0np0\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545926709
Jul 17 03:19:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:50.884Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_netstat_Udp6_RcvbufErrors\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-olap003\", endpoint_id=\"235\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"yarn_resourcemanager,zookeeper,nacos,kafka,spark,druid_data,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545929713
Jul 17 03:19:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:50.901Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_write_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/data/var/lib/kubelet/pods/3a7372b8-22f7-4934-bc38-fba12a319dfc/volumes/kubernetes.io~projected/kube-api-access-x5c6f\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545918663
Jul 17 03:19:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:50.940Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-burstable-pod132d373d_8755_45d4_97c7_04927f0acd43.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"slice\"}" timestamp=1689545926554
Jul 17 03:19:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:50.944Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"auditd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"service\"}" timestamp=1689545921057
Jul 17 03:19:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:50.959Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-0b9ea40b5cffd9ff69eecac490bb63436c5751e7768ab0bf0cad51498310c4f9.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"scope\"}" timestamp=1689545921057
Jul 17 03:19:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:51.118Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"mrzcpd_hugepages_setup.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"service\"}" timestamp=1689545926554
Jul 17 03:19:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:51.350Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx08\", asset_id=\"105\", container=\"dashboard-metrics-scraper\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Total\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545924341
Jul 17 03:19:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:51.409Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_observed_generation\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"traefik-64f55bb67d\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545921568
Jul 17 03:19:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:51.609Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_annotations\", asset=\"twa-tsgx08\", asset_id=\"105\", configmap=\"kube-root-ca.crt\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"consul\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545921568
Jul 17 03:19:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:51.657Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"52\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545926709
Jul 17 03:19:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:51.658Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"69\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545926709
Jul 17 03:19:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:51.688Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_dormant\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"virtio_dign_c\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545926709
Jul 17 03:19:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:52.435Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", container=\"coredns\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Async\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545926323
Jul 17 03:19:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:52.450Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_init_container_status_waiting\", asset=\"pcap-tsgx11\", asset_id=\"115\", container=\"firewall-init\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", pod=\"tsg-traffic-engine-numa-0-firewall-7c999f565f-xq5dg\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"93005333-2bbb-4ce3-a6b6-a5b632269832\"}" timestamp=1689545924478
Jul 17 03:19:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:52.464Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-93005333\\\\x2d2bbb\\\\x2d4ce3\\\\x2da6b6\\\\x2da5b632269832-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2dvjftg.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689545925496
Jul 17 03:19:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:52.472Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"pcap-tsgx11\", asset_id=\"115\", condition=\"false\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", pod=\"dashboard-metrics-scraper-5b94897dc9-pbxbf\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"7af362d7-f44d-40a1-9857-36884a064be9\"}" timestamp=1689545924478
Jul 17 03:19:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:52.562Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", cpu=\"4\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545929337
Jul 17 03:19:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:52.582Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_processes\", asset=\"pcap-tsgx11\", asset_id=\"115\", container=\"redis\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545915872
Jul 17 03:19:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:52.587Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_rss\", asset=\"pcap-tsgx11\", asset_id=\"115\", container=\"ipmi-exporter\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545929414
Jul 17 03:19:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:52.597Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_cpu_seconds_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"libstoragemgmt.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689545925496
Jul 17 03:19:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:52.605Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2did-wwn\\\\x2d0x62cea7f0add58c002c2c84941d8811f6\\\\x2dpart4.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689545925496
Jul 17 03:19:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:52.625Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas_updated\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"local-path-provisioner\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545924478
Jul 17 03:19:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:52.648Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_errs_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"eno36np3\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545929337
Jul 17 03:19:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:52.669Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", cpu=\"74\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545929337
Jul 17 03:19:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:52.674Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", cpu=\"33\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545929337
Jul 17 03:19:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:52.686Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"promhttp_metric_handler_requests_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", code=\"200\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"ipmi-exporter\"}" timestamp=1689545931652
Jul 17 03:19:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:52.706Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"slice\"}" timestamp=1689545925496
Jul 17 03:19:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:52.726Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-ens4f1np1.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689545925496
Jul 17 03:19:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:52.741Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_dormant\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth50cee5b3\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545929337
Jul 17 03:19:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:53.096Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"msh-tsgx08\", asset_id=\"65\", cpu=\"109\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545928328
Jul 17 03:19:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:53.123Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_down_changes_total\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"vetha6b2596c\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545928328
Jul 17 03:19:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:53.145Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx08\", asset_id=\"65\", cpu=\"72\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545928328
Jul 17 03:19:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:53.154Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", cpu=\"124\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545929337
Jul 17 03:19:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:53.198Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"overlay\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-034bf7097510c900118c7b3f234d4c5dd0887b899b0be073bc80465dc5a3393e-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689545932135
Jul 17 03:19:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:53.260Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_cpu_shares\", asset=\"msh-tsgx08\", asset_id=\"65\", container=\"exporter\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545917907
Jul 17 03:19:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:53.276Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_packets_dropped_total\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"cadvisor\", interface=\"eno1\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545916294
Jul 17 03:19:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:53.276Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_daemonset_status_number_unavailable\", asset=\"msh-tsgx08\", asset_id=\"65\", daemonset=\"prometheus-node-exporter\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545928950
Jul 17 03:19:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:53.322Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_processes\", asset=\"pcap-tsgx11\", asset_id=\"115\", container=\"kube-state-metrics\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545918939
Jul 17 03:19:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:53.323Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dracut-cmdline.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"service\"}" timestamp=1689545931179
Jul 17 03:19:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:53.422Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"nsfs\", name=\"run-netns-cni\\\\x2de7cc70ba\\\\x2d45cc\\\\x2d74f1\\\\x2d4370\\\\x2dda9adbd9faab.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689545925496
Jul 17 03:19:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:53.429Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_files_free\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", fstype=\"ext4\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/update\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545928328
Jul 17 03:19:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:53.436Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_packets_total\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"cadvisor\", interface=\"ens1f0np0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545919650
Jul 17 03:19:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:53.448Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_labels\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-node-exporter-l5pbm\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"e58d2d02-fd98-4c56-b9dd-aebd51774a57\"}" timestamp=1689545928663
Jul 17 03:19:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:53.491Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"73\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545923391
Jul 17 03:19:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:53.543Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_annotations\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"ipmi-exporter-6f8464dd94-792kb\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"da81ea3f-d797-4414-9e38-bc69ec182d3b\"}" timestamp=1689545928663
Jul 17 03:19:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:53.582Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"msh-tsgx12\", asset_id=\"113\", container=\"kubernetes-dashboard\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"cadvisor\", failure_type=\"pgmajfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"hierarchy\"}" timestamp=1689545918791
Jul 17 03:19:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:53.604Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"msh-tsgx12\", asset_id=\"113\", cpu=\"90\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545923391
Jul 17 03:19:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:53.616Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"paths.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"target\"}" timestamp=1689545931179
Jul 17 03:19:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:53.625Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.grpc.v1.cri-sandboxes-1f0a2164d250c7fdd3f11e65c867e2c9a6b0e002537de1f226bd6cf95c9c2557-shm.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689545925496
Jul 17 03:19:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:53.642Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", cpu=\"16\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545929337
Jul 17 03:19:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:53.658Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-c6171e2c08ab45cf801de20576b6cb819bb0cb1ed492b3668dad3b407d175a73.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"scope\"}" timestamp=1689545925496
Jul 17 03:19:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:53.718Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_changes_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth883fff26\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545919409
Jul 17 03:19:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:53.723Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-0e2debc9b66977ef05117d557fc158a0147f165d17003972bd5d797f6c629ca3.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"scope\"}" timestamp=1689545920884
Jul 17 03:19:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:53.736Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_frame_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth211eccff\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545919409
Jul 17 03:19:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:53.757Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2dpartlabel-EFI\\\\x5cx20System.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689545920884
Jul 17 03:19:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:53.772Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-virtual-misc-rfkill.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689545920884
Jul 17 03:19:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:53.805Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", container=\"redis\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Async\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545918444
Jul 17 03:19:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:53.817Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", container=\"dign-server-dns\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"running\"}" timestamp=1689545916464
Jul 17 03:19:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:54.006Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cpu=\"81\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545919409
Jul 17 03:19:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:54.024Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_dormant\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"vethc318cddc\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545919409
Jul 17 03:19:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:54.037Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-machine-id-commit.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689545931179
Jul 17 03:19:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:54.037Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_resident_memory_bytes\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"NetworkManager.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689545920884
Jul 17 03:19:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:54.049Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_state_started\", asset=\"msh-tsgx12\", asset_id=\"113\", container=\"prometheus-alertmanager\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-alertmanager-6df7f457b6-gvtrj\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"67eafe01-721d-41b1-93f6-903b88527edb\"}" timestamp=1689545928663
Jul 17 03:19:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:54.052Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_flags\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"vethba64267c\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545923391
Jul 17 03:19:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:54.065Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_ready\", asset=\"msh-tsgx12\", asset_id=\"113\", container=\"telegraf\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", pod=\"tsg-traffic-engine-numa-0-firewall-665bf977fd-6ss4j\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"a188b741-4451-4393-a8d5-35ceaab41710\"}" timestamp=1689545928663
Jul 17 03:19:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:54.079Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx12\", asset_id=\"113\", cpu=\"87\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545923391
Jul 17 03:19:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:54.113Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"tmpfs\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/var/lib/kubelet/pods/a70ba7d6-c83e-453e-962c-f10b0073510f/volumes/kubernetes.io~projected/kube-api-access-klfkn\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545923391
Jul 17 03:19:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:54.173Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_start_time_seconds\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689545930608
Jul 17 03:19:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:54.194Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", pod=\"kubernetes-dashboard-66dd5b456-rmvl8\", project=\"TSG-OS\", project_id=\"7\", reason=\"Evicted\", service=\"prometheus-kube-state-metrics\", uid=\"d360d8a3-9aa3-40d2-ad94-ccc470a435ce\"}" timestamp=1689545922058
Jul 17 03:19:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:54.223Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_timex_status\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545919409
Jul 17 03:19:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:54.247Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_drop_total\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"virtio_dign_s\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545923391
Jul 17 03:19:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:54.283Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_owner\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", owner_is_controller=\"true\", owner_kind=\"ReplicaSet\", owner_name=\"dashboard-metrics-scraper-5b94897dc9\", pod=\"dashboard-metrics-scraper-5b94897dc9-7xj9b\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"df708fd2-fb21-4af8-958e-4ba1dc3e2b4a\"}" timestamp=1689545928950
Jul 17 03:19:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:54.311Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"msh-tsgx08\", asset_id=\"65\", condition=\"false\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-tsg-traffic-engine-numa-1-glkhp\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"8ec229fc-9306-4096-93ee-079996a02e4e\"}" timestamp=1689545928950
Jul 17 03:19:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:54.330Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_threads\", asset=\"pcap-tsgx07\", asset_id=\"85\", container=\"telegraf\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689545913332
Jul 17 03:19:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:54.358Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx08\", asset_id=\"65\", cpu=\"96\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545928328
Jul 17 03:19:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:54.378Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_owner\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", owner_is_controller=\"true\", owner_kind=\"Deployment\", owner_name=\"prometheus-pushgateway\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"prometheus-pushgateway-bd476bcf9\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545922058
Jul 17 03:19:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:54.404Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_transmit_wait_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"mlx5_2\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545919409
Jul 17 03:19:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:54.452Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Unknown_Bit\", app_name=\"sapp4\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", vsysId=\"1\"}" timestamp=1689545927622
Jul 17 03:19:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:54.548Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_rss\", asset=\"pcap-tsgx07\", asset_id=\"85\", container=\"telegraf-security\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689545926296
Jul 17 03:19:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:54.571Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_created\", asset=\"pcap-tsgx07\", asset_id=\"85\", configmap=\"chart-content-mrzcpd-exporter\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545922058
Jul 17 03:19:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:54.623Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_resident_memory_bytes\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"rpcbind.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689545920884
Jul 17 03:19:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:54.641Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_cpu_period\", asset=\"msh-tsgx12\", asset_id=\"113\", container=\"redis\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545920589
Jul 17 03:19:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:55.169Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_frame_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth5162d954\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545919409
Jul 17 03:19:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:55.239Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_mapped_file\", asset=\"pcap-tsgx07\", asset_id=\"85\", container=\"firewall\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689545919312
Jul 17 03:19:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:55.256Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cpu=\"87\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545919409
Jul 17 03:19:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:55.274Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2did-scsi\\\\x2dSDELL_PERC_H345_Adp_000a8a508f747c2c2c00064783f0a7ce\\\\x2dpart2.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689545920884
Jul 17 03:19:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:55.306Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db13\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545925184
Jul 17 03:19:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:55.331Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"multipathd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"service\"}" timestamp=1689545920884
Jul 17 03:19:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:56.372Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_colls_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens8f0np0\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545925883
Jul 17 03:19:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:56.466Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_mapped_file\", asset=\"pcap-tsgx03\", asset_id=\"73\", container=\"prometheus-pushgateway\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545919798
Jul 17 03:19:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:56.483Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_threads_max\", asset=\"pcap-tsgx03\", asset_id=\"73\", container=\"coredns\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545927020
Jul 17 03:19:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:56.522Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_labels\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"shaping-announce-port-tsg-traffic-engine-numa-1\", uid=\"ed40b3f0-67a2-459b-afef-cc4c8849dd6a\"}" timestamp=1689545933389
Jul 17 03:19:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:56.548Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-rdma-devices-mlx5_4:1-umad.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689545923372
Jul 17 03:19:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:56.849Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Udp_Link_C2S\", app_name=\"sapp4\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", vsysId=\"1\"}" timestamp=1689545930331
Jul 17 03:19:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:56.868Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_resident_memory_bytes\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-journald.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689545923372
Jul 17 03:19:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:57.158Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"49\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545925883
Jul 17 03:19:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:57.171Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"50\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545925883
Jul 17 03:19:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:57.187Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"120\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545925883
Jul 17 03:19:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:57.272Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_file_descriptors\", asset=\"pcap-tsgx03\", asset_id=\"73\", container=\"prometheus-server-configmap-reload\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545918541
Jul 17 03:19:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:57.318Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_bytes_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"cadvisor\", interface=\"ens8f1np1\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545928255
Jul 17 03:19:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:57.354Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"pcap-tsgx03\", asset_id=\"73\", condition=\"Progressing\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"kubernetes-dashboard\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"unknown\"}" timestamp=1689545933389
Jul 17 03:19:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:57.394Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_info\", asset=\"pcap-tsgx03\", asset_id=\"73\", container=\"lb-tcp-80\", container_id=\"containerd://7eb22f8239232baff63e663a09168632214f7beb47dcdcbd21f831e822cf1996\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", image=\"docker.io/rancher/klipper-lb:v0.4.3\", image_id=\"sha256:613ae033d1f9c6950d79c1a44af3142c088bf7a419b12cc7a1fbbdbfddcc7de5\", image_spec=\"rancher/klipper-lb:v0.4.3\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"svclb-traefik-ea05df14-8sths\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"3c95424c-91e7-4b65-a797-7342b6c9f1ed\"}" timestamp=1689545933389
Jul 17 03:19:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:57.407Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-alertmanager-6df7f457b6-7bgfr\", project=\"TSG-OS\", project_id=\"7\", reason=\"Shutdown\", service=\"prometheus-kube-state-metrics\", uid=\"799dcfe0-5d9a-4a69-9a0d-087901712118\"}" timestamp=1689545933389
Jul 17 03:19:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:57.422Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_packets_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth8e70dd56\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545925883
Jul 17 03:19:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:57.427Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", phase=\"Unknown\", pod=\"mrzcpd-exporter-5cf7f967cc-q98g4\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"900223f6-7495-4cd9-b023-184522fd9a95\"}" timestamp=1689545933389
Jul 17 03:19:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:57.435Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_commands_duration_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cmd=\"client\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545926475
Jul 17 03:19:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:57.452Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_limit_bytes\", asset=\"pcap-tsgx03\", asset_id=\"73\", container=\"redis\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545931563
Jul 17 03:19:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:57.484Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"graphical.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"target\"}" timestamp=1689545923372
Jul 17 03:19:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:57.510Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"nss-lookup.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"target\"}" timestamp=1689545923372
Jul 17 03:19:58 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:58.159Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-dellr740-dt003\", asset_id=\"9\", cpu=\"4\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-dt003\", endpoint_id=\"229\", mode=\"softirq\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"hbase_master,clickhouse_data,kafka,gohangout,zookeeper,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545937407
Jul 17 03:19:58 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:58.417Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", cpu=\"19\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"OLAP-Node-Exporter-pcap-dellr740-dt001\", endpoint_id=\"251\", mode=\"nice\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545937557
Jul 17 03:19:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:59.360Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_scaling_frequency_max_hertz\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", cpu=\"22\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"OLAP-Node-Exporter-pcap-dellr740-dt001\", endpoint_id=\"251\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545937557
Jul 17 03:19:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:59.695Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_info\", address=\"56:6a:e1:ce:6f:2c\", asset=\"msh-tsgx01\", asset_id=\"51\", broadcast=\"ff:ff:ff:ff:ff:ff\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth9b710207\", duplex=\"full\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operstate=\"up\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545929083
Jul 17 03:19:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:59.711Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_multicast_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth676942d0\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545929083
Jul 17 03:19:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:59.890Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_bytes_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"cnibr0\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545929083
Jul 17 03:19:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:59.926Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_restart_policy\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-kube-state-metrics-6b768c7bf8-fnk7m\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", type=\"Always\", uid=\"75a1cd69-272c-4ee3-afeb-f27cbbc02353\"}" timestamp=1689545927425
Jul 17 03:19:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:59.931Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_bytes_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"virtio_dign_c\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545929083
Jul 17 03:19:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:19:59.960Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"9\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545929083
Jul 17 03:20:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:00.016Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"nsfs\", name=\"run-netns-cni\\\\x2ddfb025ae\\\\x2d0664\\\\x2deddf\\\\x2d11dc\\\\x2dea7820262d14.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689545924879
Jul 17 03:20:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:00.186Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_spec_paused\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"metrics-server\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545927425
Jul 17 03:20:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:00.208Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"msh-tsgx01\", asset_id=\"51\", condition=\"Available\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"local-path-provisioner\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"true\"}" timestamp=1689545927425
Jul 17 03:20:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:00.230Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_fifo_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"vethf82ad8f5\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545929083
Jul 17 03:20:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:00.249Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_carrier_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"eno35np2\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545929083
Jul 17 03:20:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:00.265Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_config_maxclients\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545925071
Jul 17 03:20:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:00.276Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_ready\", asset=\"msh-tsgx01\", asset_id=\"51\", container=\"kube-multus\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"kube-multus-ds-49nfs\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"ceaa53db-f09f-4f7d-bfb2-c311be5963c7\"}" timestamp=1689545927425
Jul 17 03:20:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:00.290Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dracut-pre-mount.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"service\"}" timestamp=1689545924879
Jul 17 03:20:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:00.302Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_metadata_resource_version\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.prometheus.v1\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545927425
Jul 17 03:20:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:00.335Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-ab3bf05bc6af66ec652e7915b0e9a2cf59933776dad043f227edea371ab24a42-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689545924879
Jul 17 03:20:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:00.418Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_daemonset_status_current_number_scheduled\", asset=\"msh-tsgx01\", asset_id=\"51\", daemonset=\"kube-multus-ds\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545927425
Jul 17 03:20:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:00.447Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"overlay\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-bc21478fdc5355bf15960d43a05074574abd6e1478c55132a193b1b21c029481-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689545924879
Jul 17 03:20:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:00.607Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_user_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545934486
Jul 17 03:20:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:00.673Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"112\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545929083
Jul 17 03:20:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:00.729Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx01\", asset_id=\"51\", container=\"coredns\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Read\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545919903
Jul 17 03:20:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:00.732Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"117\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545929083
Jul 17 03:20:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:00.746Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_packets_transmitted_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545929083
Jul 17 03:20:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:00.881Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"40\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545929083
Jul 17 03:20:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:00.909Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_speed_bytes\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens6f0np0\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545929083
Jul 17 03:20:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:00.943Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_drop_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"vethd3934cae\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545929083
Jul 17 03:20:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:00.951Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"76\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545929083
Jul 17 03:20:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:00.985Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"26\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545929083
Jul 17 03:20:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:02.452Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:a0-0000:a0:03.1-0000:a1:00.0-infiniband_mad-umad6.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689545928447
Jul 17 03:20:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:02.511Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"var-lib-kubelet-pods-03938281\\\\x2de348\\\\x2d4392\\\\x2da795\\\\x2df2d53f05d546-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2d5qkm7.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689545928447
Jul 17 03:20:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:02.592Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"117\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545938318
Jul 17 03:20:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:03.008Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"snmpd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"service\"}" timestamp=1689545928447
Jul 17 03:20:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:03.123Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", cpu=\"27\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-dt001\", endpoint_id=\"225\", mode=\"idle\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545942988
Jul 17 03:20:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:03.141Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_scaling_frequency_hertz\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", cpu=\"39\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-dt001\", endpoint_id=\"225\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545942988
Jul 17 03:20:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:03.142Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"loadmodules.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689545928447
Jul 17 03:20:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:03.152Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"net_conntrack_dialer_conn_failed_total\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", dialer_name=\"asset_ping_43\", instance=\"10.10.20.159:10090\", reason=\"resolution\", type=\"global\"}" timestamp=1689545942990
Jul 17 03:20:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:03.167Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_timex_pps_stability_hertz\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545938318
Jul 17 03:20:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:03.240Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_cache\", asset=\"msh-tsgx02\", asset_id=\"63\", container=\"telegraf\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689545921026
Jul 17 03:20:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:03.257Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_working_set_bytes\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545930997
Jul 17 03:20:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:03.268Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_created\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"metrics-server\", uid=\"4b6186fd-bece-4f61-8fbd-d18849141713\"}" timestamp=1689545931412
Jul 17 03:20:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:03.273Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"vethad44a482\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545938318
Jul 17 03:20:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:03.521Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_annotations\", asset=\"msh-tsgx02\", asset_id=\"63\", configmap=\"prometheus-alertmanager\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545931412
Jul 17 03:20:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:03.538Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx02\", asset_id=\"63\", cpu=\"36\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545938318
Jul 17 03:20:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:03.665Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"net_conntrack_dialer_conn_failed_total\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", dialer_name=\"473\", instance=\"10.10.20.159:10090\", reason=\"resolution\", type=\"global\"}" timestamp=1689545942990
Jul 17 03:20:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:06.672Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_restarts_total\", asset=\"twa-tsgx01\", asset_id=\"91\", container=\"coredns\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"coredns-6d8dc697-dlsd9\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"4237b3dc-4d91-4979-b851-594354acfc12\"}" timestamp=1689545941227
Jul 17 03:20:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:06.686Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cpu=\"73\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545943910
Jul 17 03:20:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:07.341Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_sockstat_RAW_inuse\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545943910
Jul 17 03:20:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:07.416Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_commands_rejected_calls_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cmd=\"zadd\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545938290
Jul 17 03:20:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:07.563Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db243\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545938290
Jul 17 03:20:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:07.651Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db233\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545938290
Jul 17 03:20:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:07.753Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"nfs-utils.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689545933582
Jul 17 03:20:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:07.819Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-update-utmp.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689545933582
Jul 17 03:20:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:07.833Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_dormant\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth5fb026f4\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545943910
Jul 17 03:20:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:07.859Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"clickhouse_background_message_broker_schedule_pool_task\", asset=\"msh-dellr740-olap002\", asset_id=\"13\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Clickhouse-msh-dellr740-olap002\", endpoint_id=\"73\", module=\"NC-Clickhouse\", module_id=\"29\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545947759
Jul 17 03:20:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:07.882Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"clickhouse_created_read_buffer_ordinary_total\", asset=\"msh-dellr740-olap002\", asset_id=\"13\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Clickhouse-msh-dellr740-olap002\", endpoint_id=\"73\", module=\"NC-Clickhouse\", module_id=\"29\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545947759
Jul 17 03:20:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:08.288Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_writes_bytes_total\", asset=\"twa-tsgx01\", asset_id=\"91\", container=\"dign-server-dns\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545934046
Jul 17 03:20:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:08.308Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_sockets\", asset=\"twa-tsgx01\", asset_id=\"91\", container=\"firewall\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689545942554
Jul 17 03:20:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:08.326Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_node_status_condition\", asset=\"twa-tsgx01\", asset_id=\"91\", condition=\"Ready\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", node=\"localhost\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"false\"}" timestamp=1689545941227
Jul 17 03:20:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:08.347Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_multicast_total\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth1c375d86\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545943910
Jul 17 03:20:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:08.365Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cpu=\"98\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545943910
Jul 17 03:20:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:08.413Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cpu=\"89\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545943910
Jul 17 03:20:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:08.951Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"clickhouse_arena_alloc_bytes_total\", asset=\"msh-dellr740-olap002\", asset_id=\"13\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Clickhouse-msh-dellr740-olap002\", endpoint_id=\"73\", module=\"NC-Clickhouse\", module_id=\"29\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545947759
Jul 17 03:20:09 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:09.160Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"cortex_experimental_features_in_use_total\", agent_name=\"msh-kvm-nz001\", application=\"cortex\", datacenter=\"MSH\", instance=\"127.0.0.1:10093\"}" timestamp=1689545947265
Jul 17 03:20:09 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:09.535Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_SyncTime_99th_percentile\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"WAL\"}" timestamp=1689545948804
Jul 17 03:20:09 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:09.699Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_FsReadTime_75th_percentile\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"IO\"}" timestamp=1689545948804
Jul 17 03:20:09 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:09.773Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_flushesQueuedCount\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"assessment_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"398e0a67b072d1b8b4e2adff0757660a\", table=\"index_time_assessment_hos_bucket\", type=\"index\"}" timestamp=1689545948804
Jul 17 03:20:09 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:09.911Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_compactionsCompletedCount\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"a1d7179d4b60a3d692b7220a74a8749f\", table=\"knowledge_base_hos_bucket\", type=\"data\"}" timestamp=1689545948804
Jul 17 03:20:09 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:09.961Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_readRequestCount\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"a3eb769a8c2d325e0a97838f6786a569\", table=\"session_record_hos_bucket\", type=\"data\"}" timestamp=1689545948804
Jul 17 03:20:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:10.305Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_CompactionTime_min\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Server\"}" timestamp=1689545948804
Jul 17 03:20:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:11.445Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_checkMutateFailedCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Server\"}" timestamp=1689545950149
Jul 17 03:20:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:11.585Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_replicaid\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"session_record_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"2fa4fd289d02170d9d7aaa6005053c98\", table=\"index_time_session_record_hos_bucket\", type=\"index\"}" timestamp=1689545948804
Jul 17 03:20:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:11.716Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_numReferenceFiles\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"proxy_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"3f3a8f4c87c05c3707c187b169923a9f\", table=\"index_filename_proxy_hos_bucket\", type=\"index\"}" timestamp=1689545948804
Jul 17 03:20:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:11.831Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_STATS_metric_majorCompactionOutputSize_98th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545950149
Jul 17 03:20:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:11.850Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_3_metric_compactionInputFileCount_min\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545950149
Jul 17 03:20:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:12.391Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"java_lang_GarbageCollector_LastGcInfo_memoryUsageAfterGc_committed\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", key=\"Par Eden Space\", module=\"DC-HBase\", module_id=\"3\", name=\"ParNew\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545948804
Jul 17 03:20:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:12.520Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_session_record_hos_bucket_metric_majorCompactionOutputFileCount_98th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545950149
Jul 17 03:20:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:12.539Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_default_metric_majorCompactionInputSize_75th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545950149
Jul 17 03:20:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:12.640Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_writeRequestCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"default\", project=\"OLAP\", project_id=\"3\", region=\"18f73459ccacec184acdb88b607bc5eb\", table=\"index_time_default\", type=\"index\"}" timestamp=1689545950149
Jul 17 03:20:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:12.718Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_mspan_inuse_bytes\", asset=\"twa-dellr740-dt002\", asset_id=\"39\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"OLAP-Node-Exporter-twa-dellr740-dt002\", endpoint_id=\"259\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545952055
Jul 17 03:20:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:12.753Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"twa-dellr740-dt002\", asset_id=\"39\", cpu=\"10\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"NC-Gohangout-twa-dellr740-dt002\", endpoint_id=\"179\", module=\"DC-Gohangout\", module_id=\"35\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545951525
Jul 17 03:20:13 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:13.208Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_flushesQueuedCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"c774340cb4e4e98881466697f1116f97\", table=\"default\", type=\"data\"}" timestamp=1689545950149
Jul 17 03:20:13 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:13.236Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_tsg_table_report_result_metric_scanTime_mean\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545950149
Jul 17 03:20:13 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:13.268Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"java_lang_MemoryPool_Usage_used\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"CMS Old Gen\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545950149
Jul 17 03:20:13 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:13.823Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_193_1_metric_flushTime_min\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545950149
Jul 17 03:20:13 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:13.915Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_heap_idle_bytes\", asset=\"twa-dellr740-dt002\", asset_id=\"39\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"NC-Gohangout-twa-dellr740-dt002\", endpoint_id=\"179\", module=\"DC-Gohangout\", module_id=\"35\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545951525
Jul 17 03:20:14 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:14.022Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_194_3_metric_getTime_25th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545950149
Jul 17 03:20:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:15.302Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas_ready\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"tsg-traffic-engine-numa-0-firewall\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545948076
Jul 17 03:20:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:15.305Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"iscsi-shutdown.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"oneshot\", type=\"service\"}" timestamp=1689545947046
Jul 17 03:20:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:15.354Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_owner\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", owner_is_controller=\"true\", owner_kind=\"Deployment\", owner_name=\"coredns\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"coredns-65dfb4fcf4\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545948076
Jul 17 03:20:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:15.437Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_metadata_resource_version\", asset=\"twa-tsgx07\", asset_id=\"103\", configmap=\"sapp-tsg-traffic-engine-numa-0\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545948076
Jul 17 03:20:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:15.501Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_spec_replicas\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-1-firewall-69c5b89d8c\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545948076
Jul 17 03:20:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:15.822Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"11\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545941230
Jul 17 03:20:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:15.861Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"emergency.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"target\"}" timestamp=1689545947046
Jul 17 03:20:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:15.946Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"33\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545941230
Jul 17 03:20:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:16.053Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_writes_total\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda3\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545948138
Jul 17 03:20:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:16.194Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"twa-tsgx07\", asset_id=\"103\", condition=\"unknown\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"ipmi-exporter-6f8464dd94-54bzn\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"030260b7-67cb-4efd-9b0b-ed9f7bff5dfa\"}" timestamp=1689545948076
Jul 17 03:20:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:16.418Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"0\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545941230
Jul 17 03:20:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:16.460Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"openibd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"service\"}" timestamp=1689545947046
Jul 17 03:20:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:16.562Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_ready\", asset=\"twa-tsgx07\", asset_id=\"103\", container=\"lb-tcp-443\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"svclb-traefik-b947eccc-g6kks\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"1b7e4345-c37e-48b7-8d02-089957a16f23\"}" timestamp=1689545948076
Jul 17 03:20:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:16.942Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"61\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545941230
Jul 17 03:20:17 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:17.117Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_commands_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cmd=\"unlink\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545946027
Jul 17 03:20:17 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:17.817Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"115\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545941230
Jul 17 03:20:17 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:17.847Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_owner\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", owner_is_controller=\"true\", owner_kind=\"ReplicaSet\", owner_name=\"ipmi-exporter-6f8464dd94\", pod=\"ipmi-exporter-6f8464dd94-54bzn\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"030260b7-67cb-4efd-9b0b-ed9f7bff5dfa\"}" timestamp=1689545948076
Jul 17 03:20:17 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:17.877Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"40\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545941230
Jul 17 03:20:18 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:18.239Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_start_time\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"metrics-server-54dc485875-cw4sf\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"a203ebf3-8366-4cfd-b036-168e793673ce\"}" timestamp=1689545948076
Jul 17 03:20:18 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:18.256Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"34\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545941230
Jul 17 03:20:18 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:18.305Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"88\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545941230
Jul 17 03:20:18 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:18.578Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_info\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.prometheus.v1\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545948076
Jul 17 03:20:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:19.422Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"abrt-ccpp.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"service\"}" timestamp=1689545944356
Jul 17 03:20:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:19.596Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_dormant\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"eno34np1\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545957612
Jul 17 03:20:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:19.707Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", state=\"stopped\"}" timestamp=1689545950857
Jul 17 03:20:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:19.732Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"twa-tsgx06\", asset_id=\"101\", condition=\"Available\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"systemd-exporter\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"false\"}" timestamp=1689545953949
Jul 17 03:20:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:19.752Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_type\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"chart-values-traefik\", service=\"prometheus-kube-state-metrics\", type=\"Opaque\"}" timestamp=1689545953949
Jul 17 03:20:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:20.176Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"clickhouse_disk_write_elapsed_microseconds_total\", asset=\"twa-dellr740-dt003\", asset_id=\"41\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"NC-Clickhouse-twa-dellr740-dt003\", endpoint_id=\"91\", module=\"NC-Clickhouse\", module_id=\"29\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545958300
Jul 17 03:20:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:20.533Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"22\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545948263
Jul 17 03:20:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:20.545Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"35\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545948263
Jul 17 03:20:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:20.818Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"0\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545948263
Jul 17 03:20:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:20.895Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"tmp.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689545950657
Jul 17 03:20:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:20.927Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_restarts_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", container=\"telegraf\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", pod=\"tsg-traffic-engine-numa-0-firewall-6d4969c56f-54wcn\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"617272f1-e26a-4d41-b763-554ea7e04473\"}" timestamp=1689545953828
Jul 17 03:20:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:20.927Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_bytes_total\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth9d00126e\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545957612
Jul 17 03:20:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:20.940Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"118\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545957612
Jul 17 03:20:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:20.963Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", pod=\"kubernetes-dashboard-66dd5b456-kxvwc\", project=\"TSG-OS\", project_id=\"7\", reason=\"NodeLost\", service=\"prometheus-kube-state-metrics\", uid=\"f7eb27a3-367e-4d4e-90b6-c00f5a2ffc39\"}" timestamp=1689545953828
Jul 17 03:20:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:21.044Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"57\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545948263
Jul 17 03:20:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:21.054Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_threads_max\", asset=\"twa-tsgx06\", asset_id=\"101\", container=\"metrics\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545941918
Jul 17 03:20:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:21.314Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", pod=\"tsg-traffic-engine-numa-0-firewall-5db5dbc44c-866zl\", project=\"TSG-OS\", project_id=\"7\", reason=\"NodeAffinity\", service=\"prometheus-kube-state-metrics\", uid=\"9145d365-24c7-4ff1-a1ee-0a800b33502b\"}" timestamp=1689545953949
Jul 17 03:20:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:21.626Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_flags\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens3f1np1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545948263
Jul 17 03:20:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:21.687Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_rss\", asset=\"twa-tsgx06\", asset_id=\"101\", container=\"dashboard-metrics-scraper\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545937556
Jul 17 03:20:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:21.840Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"twa-tsgx06\", asset_id=\"101\", condition=\"true\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"mrzcpd-exporter-5cf7f967cc-dxl48\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"5f8b23e9-8830-4b0a-9d61-06b46edd96f0\"}" timestamp=1689545953949
Jul 17 03:20:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:21.962Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_max_usage_bytes\", asset=\"twa-tsgx06\", asset_id=\"101\", container=\"redis\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545947762
Jul 17 03:20:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:22.009Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_packets_total\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"cadvisor\", interface=\"eth0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545952503
Jul 17 03:20:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:22.084Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_colls_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"vethdf3e6cb0\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545948263
Jul 17 03:20:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:22.423Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_annotations\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"dign-client-g7llw\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"b7bee1cc-245a-40f5-a87e-4563cb9edfda\"}" timestamp=1689545953828
Jul 17 03:20:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:22.549Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"107\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545948263
Jul 17 03:20:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:22.591Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"14\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545948263
Jul 17 03:20:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:22.847Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", cpu=\"45\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"OLAP-Node-Exporter-twa-dellr740-dt001\", endpoint_id=\"257\", mode=\"nice\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545961405
Jul 17 03:20:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:23.359Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_id\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"lo\", endpoint=\"OLAP-Node-Exporter-twa-dellr740-dt001\", endpoint_id=\"257\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545961405
Jul 17 03:20:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:23.470Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_excessive_buffer_overrun_errors_total\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_2\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545954790
Jul 17 03:20:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:23.487Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_down_changes_total\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens8f0np0\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545954790
Jul 17 03:20:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:23.502Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"msh-tsgx09\", asset_id=\"55\", condition=\"Progressing\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"cm-policy-local-cache-redis-master\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"true\"}" timestamp=1689545960068
Jul 17 03:20:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:23.508Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db110\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545957019
Jul 17 03:20:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:23.538Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-dellr740-olap004\", asset_id=\"17\", cpu=\"13\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Spark-msh-dellr740-olap004\", endpoint_id=\"15\", mode=\"iowait\", module=\"NC-Spark\", module_id=\"47\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545962253
Jul 17 03:20:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:23.541Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx09\", asset_id=\"55\", cpu=\"106\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545954790
Jul 17 03:20:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:23.577Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx09\", asset_id=\"55\", cpu=\"94\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545954790
Jul 17 03:20:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:23.599Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"overlay\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-83286f721fcb3123fe7ecffc53f05572b024bd12ed7f95ba49a4871f75d356cb-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689545958416
Jul 17 03:20:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:23.786Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Succeeded\", pod=\"helm-install-systemd-exporter-2h74t\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"75668a8f-85fc-45c3-9b22-ebd6bba21c17\"}" timestamp=1689545960068
Jul 17 03:20:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:23.802Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"1\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545954790
Jul 17 03:20:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:23.900Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_info\", asset=\"msh-tsgx09\", asset_id=\"55\", cluster_ip=\"10.43.13.92\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"kubernetes-dashboard\", uid=\"74413472-e3ff-44d4-b9a6-c1c4ec5b112d\"}" timestamp=1689545960068
Jul 17 03:20:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:23.990Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_multicast_packets_received_total\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_5\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545954790
Jul 17 03:20:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:24.126Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"4\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545960160
Jul 17 03:20:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:24.147Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.grpc.v1.cri-sandboxes-2cecd16a141d9d68b235173be7c3e8115692fc6e05b6fdb1d80145067687e261-shm.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689545951102
Jul 17 03:20:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:24.170Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"54\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545960160
Jul 17 03:20:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:24.212Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db96\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545961795
Jul 17 03:20:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:24.267Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_virtual_memory_max_bytes\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"packagekit.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689545951102
Jul 17 03:20:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:24.323Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2did-scsi\\\\x2d362cea7f0add596002c2c8d93a97a624b\\\\x2dpart1.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689545951102
Jul 17 03:20:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:24.628Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_working_set_bytes\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689545960991
Jul 17 03:20:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:24.659Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_flags\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"cnibr0\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545954790
Jul 17 03:20:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:24.693Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_labels\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-delete-tsg-traffic-engine-numa-1-bjvt6\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"55e15df4-2d3f-4878-94dc-5fef3e71fbdb\"}" timestamp=1689545960068
Jul 17 03:20:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:24.717Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_info\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_endpoint=\"kubernetes-dashboard\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545951122
Jul 17 03:20:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:24.780Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx09\", asset_id=\"55\", cpu=\"19\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545954790
Jul 17 03:20:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:25.108Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"76\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545960160
Jul 17 03:20:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:25.150Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_protocol_type\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"vethf6db1ea3\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545960160
Jul 17 03:20:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:25.155Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_spec_replicas\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"systemd-exporter-7f549d9485\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545951122
Jul 17 03:20:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:25.172Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db17\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545961795
Jul 17 03:20:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:25.230Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"process_virtual_memory_bytes\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545961795
Jul 17 03:20:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:25.301Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:00-0000:00:01.1-0000:01:00.1-net-ens1f1np1.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689545951102
Jul 17 03:20:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:25.559Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"pcap-tsgx12\", asset_id=\"117\", condition=\"unknown\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"systemd-exporter-7f549d9485-m5dr9\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"0557366f-a93d-40e0-ba86-42e3b764cacf\"}" timestamp=1689545951122
Jul 17 03:20:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:25.887Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_replicas\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"configmap-reloader-6c55bbc49f\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545951122
Jul 17 03:20:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:25.980Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"110\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545960160
Jul 17 03:20:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:26.003Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"iscsi.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689545951102
Jul 17 03:20:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:26.086Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_max_usage_bytes\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"configmap-reloader\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545960015
Jul 17 03:20:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:26.131Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_daemonset_status_number_unavailable\", asset=\"pcap-tsgx12\", asset_id=\"117\", daemonset=\"dign-client\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545951122
Jul 17 03:20:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:26.156Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"java_lang_MemoryPool_UsageThresholdSupported\", asset=\"pcap-dellr740-dt003\", asset_id=\"35\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-Yarn-pcap-dellr740-dt003-NodeManager\", endpoint_id=\"361\", module=\"DC-Yarn\", module_id=\"1\", name=\"Compressed Class Space\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545965525
Jul 17 03:20:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:26.425Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_metadata_resource_version\", asset=\"twa-tsgx09\", asset_id=\"107\", configmap=\"chart-content-traefik-crd\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545958744
Jul 17 03:20:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:26.515Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_labels\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.prometheus.v1\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545958744
Jul 17 03:20:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:26.532Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_slowlog_last_id\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545963549
Jul 17 03:20:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:26.842Z caller=dedupe.go:112 component=remote level=error remote_name=52db6f url=http://10.10.10.159:10090/nz-agent/prometheus/proxy/api/v1/write msg="non-recoverable error" count=1000 exemplarCount=0 err="server returned HTTP status 400 : out of order sample"
Jul 17 03:20:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:27.034Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-pod8abd475c_c69e_4e96_9ca6_1c2e74278a42.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"slice\"}" timestamp=1689545953907
Jul 17 03:20:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:27.168Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_labels\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"kubernetes\", uid=\"b9800000-8bd5-4bc3-9913-90efe32b01c4\"}" timestamp=1689545958744
Jul 17 03:20:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:27.185Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"61\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545955810
Jul 17 03:20:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:27.258Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_restart_policy\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-tsg-traffic-engine-numa-0-85495\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", type=\"OnFailure\", uid=\"32092806-1b0c-44f8-8750-8d654c41d7a9\"}" timestamp=1689545958744
Jul 17 03:20:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:27.384Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_scrape_collector_duration_seconds\", asset=\"twa-tsgx09\", asset_id=\"107\", collector=\"ipvs\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545955810
Jul 17 03:20:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:27.425Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Failed\", pod=\"helm-install-traefik-rdmft\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"3daa5d94-8d7c-4d71-883e-255619668328\"}" timestamp=1689545955682
Jul 17 03:20:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:27.489Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_drop_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens2f1np1\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545962882
Jul 17 03:20:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:27.503Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-fs-fuse-connections.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689545954114
Jul 17 03:20:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:27.739Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_annotations\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_endpoint=\"shaping-announce-port-tsg-traffic-engine-numa-0\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545955682
Jul 17 03:20:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:27.769Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_info\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"localhost.node-password.k3s\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545955682
Jul 17 03:20:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:27.776Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-ttyS0.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689545954114
Jul 17 03:20:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:27.804Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"75\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545962882
Jul 17 03:20:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:27.812Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"shm\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/d0eaf79bd8f0fdf3354476645a85ca968217972ef9030249453a97f71ad68165/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545962882
Jul 17 03:20:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:27.820Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"kube-state-metrics\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Sync\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545957694
Jul 17 03:20:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:27.848Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_compressed_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens3f1np1\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545962882
Jul 17 03:20:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:27.854Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_annotations\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"dign-server-pf96k\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"75dc4160-da0a-4767-823a-227b8dd303c1\"}" timestamp=1689545955682
Jul 17 03:20:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:27.895Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"3\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545962882
Jul 17 03:20:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:27.917Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"117\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545962882
Jul 17 03:20:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:27.932Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-848571bc\\\\x2da31e\\\\x2d44de\\\\x2da0f0\\\\x2dd172cf8f3bd0-volume\\\\x2dsubpaths-telegraf-telegraf\\\\x2dsecurity-0.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689545954114
Jul 17 03:20:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:27.993Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_labels\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_endpoint=\"kube-dns\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545955682
Jul 17 03:20:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:28.010Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_status_active\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", job_name=\"helm-install-systemd-exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545955682
Jul 17 03:20:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:28.021Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens7f0np0\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545962882
Jul 17 03:20:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:28.859Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"56\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545962882
Jul 17 03:20:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:28.894Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"32\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545962882
Jul 17 03:20:29 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:29.745Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"loki_response_message_bytes_bucket\", agent_name=\"msh-kvm-nz001\", application=\"loki\", datacenter=\"MSH\", instance=\"127.0.0.1:13100\", le=\"1.048576e+07\", method=\"gRPC\", route=\"/grpc.health.v1.Health/Check\"}" timestamp=1689545968328
Jul 17 03:20:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:31.681Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"89\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545959093
Jul 17 03:20:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:31.760Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"cm-policy-local-cache-redis-master-69967b5879-ttmw8\", project=\"TSG-OS\", project_id=\"7\", reason=\"NodeLost\", service=\"prometheus-kube-state-metrics\", uid=\"1ae789f3-8760-4836-9727-170790799776\"}" timestamp=1689545959276
Jul 17 03:20:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:31.795Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"81\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545960791
Jul 17 03:20:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:31.795Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"overlay\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-1ef773a86a4a57c19fc87996aed59dd63c624d5fae04b097728a17b29ac88dbb-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689545962644
Jul 17 03:20:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:31.816Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link_mode\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"eno34np1\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545960791
Jul 17 03:20:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:31.832Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"83\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545960791
Jul 17 03:20:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:32.188Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"ldconfig.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"service\"}" timestamp=1689545966907
Jul 17 03:20:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:32.208Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"mlocate-updatedb.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"service\"}" timestamp=1689545966907
Jul 17 03:20:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:32.218Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-95d5b60888ba291b4d576ad8bf0b3f4ea40c0dc6b50f0b3efcf92f20d93cc451.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"scope\"}" timestamp=1689545961015
Jul 17 03:20:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:32.469Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"twa-tsgx03\", asset_id=\"95\", condition=\"true\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-server-6ff784d758-tq6lp\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"1a454423-a4c8-40de-896e-6655fed1aa68\"}" timestamp=1689545970436
Jul 17 03:20:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:32.514Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_limit_bytes\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/shm\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545955586
Jul 17 03:20:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:32.543Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"var-lib-kubelet-pods-5eafbc6c\\\\x2d7214\\\\x2d4748\\\\x2da907\\\\x2d76b294823901-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2dq8vz5.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689545967691
Jul 17 03:20:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:32.589Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"abrt-journal-core.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"service\"}" timestamp=1689545961015
Jul 17 03:20:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:32.593Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-f4180b3bccf6773c17e325b9e8fb303ddbf61c40593cb8607bc15ea65d1ccc2c-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689545967691
Jul 17 03:20:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:32.723Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_colls_total\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethbe41d052\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545960605
Jul 17 03:20:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:32.736Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"76\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545960791
Jul 17 03:20:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:32.832Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_bytes_total\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethfd281763\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545959093
Jul 17 03:20:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:32.846Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_fifo_total\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens2f0np0\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545959093
Jul 17 03:20:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:32.887Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_owner\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", owner_is_controller=\"true\", owner_kind=\"ReplicaSet\", owner_name=\"coredns-58679555bf\", pod=\"coredns-58679555bf-v6dn2\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"8c5c1473-ca58-435f-a331-a241afdff2f9\"}" timestamp=1689545959894
Jul 17 03:20:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:32.899Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"119\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545960791
Jul 17 03:20:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:32.932Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-df0a65bb34ddb9c67c5a1c08ef9fcd3449025d6dd44fdee7a4db87d244abedab.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"scope\"}" timestamp=1689545962644
Jul 17 03:20:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:32.963Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_type\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.tsg-traffic-engine-numa-1.v1\", service=\"prometheus-kube-state-metrics\", type=\"helm.sh/release.v1\"}" timestamp=1689545959894
Jul 17 03:20:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:32.989Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_carrier_total\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens2f0np0\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545960791
Jul 17 03:20:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:33.033Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"65\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545960605
Jul 17 03:20:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:33.038Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545959093
Jul 17 03:20:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:33.063Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-da4fef9f\\\\x2d6e38\\\\x2d4557\\\\x2dac6b\\\\x2d3ab5eb04a34f-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2dgdwj7.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689545966907
Jul 17 03:20:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:33.099Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_readonly\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"shm\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/e4ba4b14a19c466b5a530124fe01c796f501f2cbf6ffef19efc3407f0bac587d/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545960605
Jul 17 03:20:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:33.328Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_memory_Active_file_bytes\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545960791
Jul 17 03:20:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:33.344Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"102\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545959093
Jul 17 03:20:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:33.359Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"twa-tsgx03\", asset_id=\"95\", container=\"exporter\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"iowaiting\"}" timestamp=1689545953486
Jul 17 03:20:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:33.954Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_threads\", asset=\"twa-tsgx05\", asset_id=\"99\", container=\"prometheus-server-configmap-reload\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545944468
Jul 17 03:20:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:33.969Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"10\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545960605
Jul 17 03:20:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:33.986Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"52\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545960605
Jul 17 03:20:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:34.029Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"26\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545960605
Jul 17 03:20:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:34.041Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"36\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545960605
Jul 17 03:20:35 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:35.802Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"java_lang_GarbageCollector_LastGcInfo_memoryUsageAfterGc_committed\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-Hmaster\", endpoint_id=\"347\", key=\"Par Eden Space\", module=\"DC-HBase\", module_id=\"3\", name=\"ParNew\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545974849
Jul 17 03:20:36 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:36.113Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kafka_server_BrokerTopicMetrics_OneMinuteRate\", asset=\"pcap-dellr740-dt002\", asset_id=\"33\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-Kafka-pcap-dellr740-dt002\", endpoint_id=\"101\", module=\"DC-Kafka\", module_id=\"5\", name=\"ProduceMessageConversionsPerSec\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", topic=\"INTERNAL-PACKET-CAPTURE-EVENT\"}" timestamp=1689545975787
Jul 17 03:20:39 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:39.323Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-kvm-cm001\", asset_id=\"3\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db44\", endpoint=\"CM-Branch-Redis-msh-kvm-cm001\", endpoint_id=\"319\", host=\"192.168.20.157\", module=\"CM-Branch-Redis\", module_id=\"73\", nz_agent_id=\"3\", parent_asset=\"msh-dellr740-pve001\", parent_asset_id=\"1\", project=\"Central-Management\", project_id=\"5\"}" timestamp=1689545978579
Jul 17 03:20:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:40.375Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"32\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545971647
Jul 17 03:20:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:40.417Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_tasks_max\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-ask-password-wall.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689545975764
Jul 17 03:20:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:40.459Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"68\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545971647
Jul 17 03:20:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:40.576Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_bytes_total\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens4f1np1\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545971647
Jul 17 03:20:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:40.719Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_multicast_total\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth607a2032\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545971647
Jul 17 03:20:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:40.786Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"twa-tsgx04\", asset_id=\"97\", container=\"lb-tcp-80\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"running\"}" timestamp=1689545950057
Jul 17 03:20:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:40.953Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_restarts_total\", asset=\"twa-tsgx04\", asset_id=\"97\", container=\"prometheus-node-exporter\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-node-exporter-xrmbl\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"9f402370-116a-4e69-9fa9-b0192f6cd502\"}" timestamp=1689545980034
Jul 17 03:20:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:41.163Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db32\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545973827
Jul 17 03:20:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:41.205Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_CATALOG_metric_putTime_median\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545979756
Jul 17 03:20:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:41.242Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_10_194_1_metric_putTime_num_ops\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545979756
Jul 17 03:20:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:41.275Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_deleteCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"knowledge_base_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"7a86d8decc01e6ad604ab256b943d227\", table=\"index_time_knowledge_base_hos_bucket\", type=\"index\"}" timestamp=1689545979756
Jul 17 03:20:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:41.337Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Replay_25th_percentile\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Server\"}" timestamp=1689545979756
Jul 17 03:20:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:41.368Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_deleteCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"knowledge_base_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"04d699da099bd2e5815e410973cf631a\", table=\"index_time_knowledge_base_hos_bucket\", type=\"index\"}" timestamp=1689545979756
Jul 17 03:20:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:41.376Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_avgStoreFileAge\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"troubleshooting_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"2bb40fcd716437b4939ab03c58064dd2\", table=\"index_time_troubleshooting_hos_bucket\", type=\"index\"}" timestamp=1689545979756
Jul 17 03:20:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:41.525Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_config_maxclients\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545973827
Jul 17 03:20:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:41.576Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db180\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545973827
Jul 17 03:20:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:41.597Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_cpu_seconds_total\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"hasp_monitor.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689545975764
Jul 17 03:20:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:42.040Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_packets_total\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"cadvisor\", interface=\"ens8f1np1\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545968741
Jul 17 03:20:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:42.060Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"104\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545971647
Jul 17 03:20:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:42.096Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-machine-id-commit.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"oneshot\", type=\"service\"}" timestamp=1689545975764
Jul 17 03:20:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:42.319Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_dormant\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens7f0np0\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545971647
Jul 17 03:20:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:42.488Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_info\", asset=\"pcap-tsgx01\", asset_id=\"71\", board_id=\"MT_0000000359\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"mlx5_4\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", firmware_version=\"22.28.4000\", hca_type=\"MT4125\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545976242
Jul 17 03:20:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:42.518Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_bytes_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"br_dign_s\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545976242
Jul 17 03:20:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:42.587Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Tcp_Lost_Bit\", app_name=\"sapp4\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", vsysId=\"1\"}" timestamp=1689545967581
Jul 17 03:20:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:42.605Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"mdmonitor.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689545980671
Jul 17 03:20:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:42.704Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", cpu=\"88\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545976242
Jul 17 03:20:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:42.716Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Udp_Link_S2C_ALL\", app_name=\"sapp4\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", vsysId=\"1\"}" timestamp=1689545969178
Jul 17 03:20:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:42.727Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"120\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545976242
Jul 17 03:20:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:42.744Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", cpu=\"126\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545976242
Jul 17 03:20:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:42.766Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"scrape_duration_seconds\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_instance=\"10.42.0.52:9290\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"ipmi-exporter\"}" timestamp=1689545967619
Jul 17 03:20:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:42.786Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_resident_memory_bytes\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"smartd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689545980671
Jul 17 03:20:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:42.792Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db121\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545974838
Jul 17 03:20:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:42.882Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"abrt-vmcore.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"oneshot\", type=\"service\"}" timestamp=1689545980671
Jul 17 03:20:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:42.883Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_labels\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"kubernetes\", uid=\"ea290d21-ce59-4b37-9f9f-1763710c736f\"}" timestamp=1689545975154
Jul 17 03:20:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:42.884Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_virtual_memory_max_bytes\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"chronyd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689545980671
Jul 17 03:20:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:42.945Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_annotations\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"prometheus-pushgateway-bd476bcf9\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545975154
Jul 17 03:20:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:42.965Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled_time\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"metrics-server-54dc485875-dfwgq\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"d1cd38b9-775b-4294-ac65-580f7b9cb70e\"}" timestamp=1689545975154
Jul 17 03:20:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:43.023Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", cpu=\"6\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545976242
Jul 17 03:20:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:43.057Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_observed_generation\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-1-firewall-6b6bf4679c\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545975154
Jul 17 03:20:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:43.074Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_memory_WritebackTmp_bytes\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545976242
Jul 17 03:20:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:43.123Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", cpu=\"66\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545976242
Jul 17 03:20:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:43.139Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", cpu=\"9\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545976242
Jul 17 03:20:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:43.237Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_vl15_dropped_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"mlx5_3\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545976242
Jul 17 03:20:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:43.253Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_annotations\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-cm-policy-local-cache-7kss5\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"4d79d568-e597-4292-9848-bc8f41715e7a\"}" timestamp=1689545975154
Jul 17 03:20:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:43.306Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"blk-availability.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689545980671
Jul 17 03:20:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:43.586Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", state=\"sleeping\"}" timestamp=1689545980075
Jul 17 03:20:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:43.595Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_gc_duration_seconds_sum\", asset=\"msh-dellr740-olap002\", asset_id=\"13\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Mariadb-msh-dellr740-olap002\", endpoint_id=\"69\", module=\"NC-Mariadb\", module_id=\"43\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545982786
Jul 17 03:20:45 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:45.292Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_memory_SwapTotal_bytes\", asset=\"msh-kvm-cm001\", asset_id=\"3\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"CM-Node-Exporter-msh-kvm-cm001\", endpoint_id=\"391\", module=\"CM-Node-Exporter\", module_id=\"75\", nz_agent_id=\"3\", parent_asset=\"msh-dellr740-pve001\", parent_asset_id=\"1\", project=\"Central-Management\", project_id=\"5\"}" timestamp=1689545984235
Jul 17 03:20:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:49.469Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_compactionsQueuedCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"firewall_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"64070c81c737589d75cf08334714566f\", table=\"index_time_firewall_hos_bucket\", type=\"index\"}" timestamp=1689545987438
Jul 17 03:20:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:49.481Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_getCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"d42dd43fc4f852d90fce6a89a6ff9936\", table=\"rtp_hos_bucket\", type=\"data\"}" timestamp=1689545987438
Jul 17 03:20:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:49.492Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_default_metric_majorCompactionInputFileCount_max\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689545987438
Jul 17 03:20:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:49.508Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_tsg_galaxy_table_relation_user_teid_metric_scanTime_99th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545987438
Jul 17 03:20:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:49.516Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_numBytesCompactedCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"knowledge_base_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"ea7ddc73e094170bd7b8d6b4fe2226a8\", table=\"index_filename_knowledge_base_hos_bucket\", type=\"index\"}" timestamp=1689545987438
Jul 17 03:20:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:49.575Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_maxFlushQueueSize\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"proxy_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"f0123c96ecf2d946605d86423433d8fc\", table=\"index_partfile_proxy_hos_bucket\", type=\"index\"}" timestamp=1689545987438
Jul 17 03:20:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:49.831Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_flushesQueuedCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"assessment_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"ec47909d48a12c9b2d848e3ffe247d2d\", table=\"index_time_assessment_hos_bucket\", type=\"index\"}" timestamp=1689545987438
Jul 17 03:20:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:49.846Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_replicaid\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"assessment_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"c9d7907cec295352aecb567b36e66792\", table=\"index_filename_assessment_hos_bucket\", type=\"index\"}" timestamp=1689545987438
Jul 17 03:20:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:49.862Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx10\", asset_id=\"109\", cpu=\"9\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545981797
Jul 17 03:20:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:49.964Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_1_metric_incrementTime_75th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545987438
Jul 17 03:20:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:49.968Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", fstype=\"ext4\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/data/var/lib/kubelet/pods/ba106c44-d5c3-446a-b10b-530b2492f73f/volume-subpaths/config-volume/telegraf-security/1\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545981797
Jul 17 03:20:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:49.984Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_194_2_metric_incrementTime_99_9th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689545987438
Jul 17 03:20:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:50.104Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_node_labels\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", node=\"localhost\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545979668
Jul 17 03:20:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:50.180Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_packets_total\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", interface=\"ens2f1np1\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545975461
Jul 17 03:20:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:50.182Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_swap\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"lb-tcp-443\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545977783
Jul 17 03:20:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:50.220Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx10\", asset_id=\"109\", cpu=\"36\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545981797
Jul 17 03:20:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:50.247Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_down_changes_total\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethcddab70d\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545981797
Jul 17 03:20:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:50.375Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx10\", asset_id=\"109\", cpu=\"32\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545981797
Jul 17 03:20:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:50.918Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_state_started\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"helm\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-ipmi-exporter-bbkqn\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"af48b541-cb04-486f-951a-b49f9ad44dd5\"}" timestamp=1689545979668
Jul 17 03:20:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:50.951Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"nfs-mountd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"service\"}" timestamp=1689545981057
Jul 17 03:20:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:50.979Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"tsg_master_T_success_log\", app_name=\"tsg_master\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"traffic-engine\", line_name=\"GTPC-RECORD\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", table_name=\"sendlog\", vsysId=\"1\"}" timestamp=1689545982447
Jul 17 03:20:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:51.020Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-8bb79bea\\\\x2d895f\\\\x2d4fbf\\\\x2dbba7\\\\x2dff346d91b1a0-volume\\\\x2dsubpaths-config\\\\x2dvolume-telegraf\\\\x2dsecurity-1.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689545981057
Jul 17 03:20:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:51.179Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"configmap-reloader\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Read\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545968923
Jul 17 03:20:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:51.344Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"twa-tsgx08\", asset_id=\"105\", container=\"telegraf-security\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", failure_type=\"pgfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"hierarchy\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689545984147
Jul 17 03:20:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:51.372Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_labels\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-mrzcpd-exporter-zjhth\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"a88ca435-a295-43d3-bffd-a3842ce86c5e\"}" timestamp=1689545981568
Jul 17 03:20:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:51.502Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_reservation_limit_bytes\", asset=\"twa-tsgx08\", asset_id=\"105\", container=\"telegraf-security\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689545987561
Jul 17 03:20:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:51.542Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_annotations\", asset=\"twa-tsgx08\", asset_id=\"105\", configmap=\"chart-content-tsg-traffic-engine-numa-0\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545981568
Jul 17 03:20:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:51.556Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_max_fds\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"crond.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689545986554
Jul 17 03:20:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:51.557Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_address_available\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_endpoint=\"cm-policy-local-cache-redis-headless\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545981568
Jul 17 03:20:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:51.854Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_writes_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"overlay_0-288\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545979009
Jul 17 03:20:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:51.871Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_scaling_frequency_hertz\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"39\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545986709
Jul 17 03:20:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:51.882Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_local_link_integrity_errors_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_0\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545986709
Jul 17 03:20:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:52.425Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", cpu=\"99\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545989337
Jul 17 03:20:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:52.828Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_device_id\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens7f1np1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545989337
Jul 17 03:20:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:52.852Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db89\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545979367
Jul 17 03:20:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:52.868Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_defrag_key_misses\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545979367
Jul 17 03:20:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:53.136Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_fifo_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth2169720f\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545989337
Jul 17 03:20:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:53.171Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", cpu=\"3\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545989337
Jul 17 03:20:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:53.242Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", cpu=\"113\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545989337
Jul 17 03:20:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:53.276Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Write\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689545984572
Jul 17 03:20:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:53.285Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_limit_bytes\", asset=\"pcap-tsgx11\", asset_id=\"115\", container=\"exporter-merger\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545990039
Jul 17 03:20:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:53.295Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failcnt\", asset=\"pcap-tsgx11\", asset_id=\"115\", container=\"local-path-provisioner\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545983285
Jul 17 03:20:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:53.305Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"msh-tsgx08\", asset_id=\"65\", container=\"telegraf-security\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"cadvisor\", failure_type=\"pgfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"hierarchy\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689545968188
Jul 17 03:20:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:53.307Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_swap_limit_bytes\", asset=\"pcap-tsgx11\", asset_id=\"115\", container=\"ipmi-exporter\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545990039
Jul 17 03:20:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:53.316Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas_ready\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"coredns\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545984478
Jul 17 03:20:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:53.335Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", cpu=\"103\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545989337
Jul 17 03:20:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:53.378Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"overlay\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-d4f54b6dfb405dee3bbcc323716d834277dee6d5d263ce80a080e0ff522f3272-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689545991179
Jul 17 03:20:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:53.388Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_spec_strategy_rollingupdate_max_surge\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"configmap-reloader\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545988950
Jul 17 03:20:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:53.398Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"shm\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/cad7e80d2f33cd9403f05ebc02c7041d9f5135e6996cf09da95bf4a6340fd257/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545989337
Jul 17 03:20:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:53.414Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_drop_total\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens8f0np0\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545988328
Jul 17 03:20:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:53.419Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-coredump@0.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"simple\", type=\"service\"}" timestamp=1689545985496
Jul 17 03:20:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:53.462Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_spec_completions\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", job_name=\"helm-install-configmap-reloader\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545988663
Jul 17 03:20:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:53.484Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_info\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.traefik.v1\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545984478
Jul 17 03:20:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:53.533Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_packets_total\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens8f0np0\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545988328
Jul 17 03:20:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:53.564Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-infiniband-umad0.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689545992135
Jul 17 03:20:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:53.853Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_writes_total\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545972645
Jul 17 03:20:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:53.862Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"87\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545988328
Jul 17 03:20:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:53.952Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_virtual_memory_max_bytes\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-logind.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689545985496
Jul 17 03:20:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:53.985Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-netns-cni\\\\x2d33d40af5\\\\x2dfd1d\\\\x2df2a5\\\\x2d5e82\\\\x2d8674c384cbe1.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689545985496
Jul 17 03:20:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:54.063Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"tsg_master_shaping\", app_name=\"tsg_master\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"traffic-engine\", line_name=\"activing_fail\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", table_name=\"ctrl_service\", vsysId=\"1\"}" timestamp=1689545984735
Jul 17 03:20:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:54.106Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cpu=\"96\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545979409
Jul 17 03:20:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:54.243Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"tmpfs\", name=\"data-var-lib-kubelet-pods-1553814d\\\\x2d54e0\\\\x2d4f9e\\\\x2db251\\\\x2dbfcf1ade925f-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2dxvl2l.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689545985496
Jul 17 03:20:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:54.287Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"ck_merges_count\", asset=\"pcap-dellr740-dt002\", asset_id=\"33\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"NC-Gohangout-pcap-dellr740-dt002\", endpoint_id=\"173\", module=\"DC-Gohangout\", module_id=\"35\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545992666
Jul 17 03:20:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:54.319Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"cadvisor\", failure_type=\"pgfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"container\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689545977252
Jul 17 03:20:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:54.411Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_multicast_total\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"eno33np0\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545988328
Jul 17 03:20:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:54.451Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"85\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545988328
Jul 17 03:20:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:54.481Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"system-systemd\\\\x2dcoredump.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"slice\"}" timestamp=1689545980884
Jul 17 03:20:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:54.514Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_annotations\", asset=\"pcap-tsgx11\", asset_id=\"115\", configmap=\"local-path-config\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545984478
Jul 17 03:20:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:54.525Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"shm\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/7512c673cf7ffa28dd5a904fa1484f8255f3c895be156a23d940681ab8b297a3/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545988328
Jul 17 03:20:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:54.533Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_address_assign_type\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"virtio_dign_s\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545988328
Jul 17 03:20:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:54.539Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_tasks_max\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-firstboot.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689545985496
Jul 17 03:20:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:54.558Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_down_changes_total\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens4f3np3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545988328
Jul 17 03:20:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:54.562Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db8\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545990108
Jul 17 03:20:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:54.797Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_receive_remote_physical_errors_total\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_0\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545988328
Jul 17 03:20:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:54.929Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_address_not_ready\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_endpoint=\"shaping-announce-port-tsg-traffic-engine-numa-0\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545988663
Jul 17 03:20:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:54.948Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_labels\", asset=\"msh-tsgx12\", asset_id=\"113\", configmap=\"local-path-config\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545988663
Jul 17 03:20:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:54.972Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_avail_bytes\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"shm\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/cbfd93d74de91da2efaf980c1fead763d212f23c3778fc03fe7fbec5875b94fe/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545988328
Jul 17 03:20:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:55.065Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"50\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545983391
Jul 17 03:20:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:55.095Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"var-lib-kubelet-pods-39dc30a6\\\\x2d78ff\\\\x2d4c95\\\\x2d95b1\\\\x2dcfaeff1800c6-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2dwbfv2.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689545992135
Jul 17 03:20:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:55.122Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_cache\", asset=\"msh-tsgx12\", asset_id=\"113\", container=\"prometheus-alertmanager\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545963183
Jul 17 03:20:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:55.126Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"msh-tsgx12\", asset_id=\"113\", container=\"firewall\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"cadvisor\", failure_type=\"pgfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"container\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689545961712
Jul 17 03:20:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:55.143Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_threads\", asset=\"msh-tsgx12\", asset_id=\"113\", container=\"traefik\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545978375
Jul 17 03:20:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:55.181Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_packets_dropped_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"cadvisor\", interface=\"bond0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545988429
Jul 17 03:20:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:55.201Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_restart_policy\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-traefik-crd-ldddg\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", type=\"OnFailure\", uid=\"e773b3b3-1ae7-46d9-a705-f7b732c0c435\"}" timestamp=1689545988663
Jul 17 03:20:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:55.248Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx08\", asset_id=\"65\", cpu=\"85\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545988328
Jul 17 03:20:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:55.248Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-infiniband-umad5.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689545991179
Jul 17 03:20:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:55.259Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_info\", asset=\"msh-tsgx12\", asset_id=\"113\", container=\"helm\", container_id=\"containerd://53478ffa8133073a7a1905eda49248bc0c366f46340343dc05807f090aeed3c2\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", image=\"docker.io/rancher/klipper-helm:v0.8.0-build20230510\", image_id=\"sha256:6f42df210d7faa0b6eb56f5f28bb061930c7834af1e7dd4b3586108ec6b1485b\", image_spec=\"rancher/klipper-helm:v0.8.0-build20230510\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-cm-policy-local-cache-bqfl6\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"8e29cfbc-c73d-46b2-97c0-7a5b8af76f8a\"}" timestamp=1689545988663
Jul 17 03:20:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:55.264Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db41\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545990108
Jul 17 03:20:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:55.270Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx12\", asset_id=\"113\", cpu=\"14\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545983391
Jul 17 03:20:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:55.290Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"emergency.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"target\"}" timestamp=1689545991179
Jul 17 03:20:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:55.297Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_labels\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"coredns-6cb546fb97\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545988663
Jul 17 03:20:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:55.318Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_spec_parallelism\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", job_name=\"helm-install-mrzcpd-exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545988663
Jul 17 03:20:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:55.363Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-netns-cni\\\\x2d931c816a\\\\x2d425f\\\\x2d33aa\\\\x2d6501\\\\x2d96257089ebab.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689545991179
Jul 17 03:20:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:55.479Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2duuid-eb3ef54c\\\\x2de946\\\\x2d46ea\\\\x2db82c\\\\x2d3ff03e40f719.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689545980884
Jul 17 03:20:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:55.575Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_errors_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"cadvisor\", interface=\"ens7f0np0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545988429
Jul 17 03:20:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:55.588Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cpu=\"74\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545979409
Jul 17 03:20:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:55.606Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_changes_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens4f2np2\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545979409
Jul 17 03:20:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:55.778Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_virtual_memory_max_bytes\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"snmpd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689545980884
Jul 17 03:20:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:55.814Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-b4e1ea9a1bdf46e1c045aa09601605abd5c1154bd72d8029126fbd9ca093e129.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"scope\"}" timestamp=1689545980884
Jul 17 03:20:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:56.221Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"pcap-tsgx03\", asset_id=\"73\", condition=\"false\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-traefik-9tnn5\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"838f1313-6f16-4baf-817b-719f97e6d63c\"}" timestamp=1689545993389
Jul 17 03:20:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:56.234Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_unicast_packets_transmitted_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"mlx5_8\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545985883
Jul 17 03:20:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:56.255Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_compressed_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"cnibr0\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545985883
Jul 17 03:20:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:56.294Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"import-state.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689545983372
Jul 17 03:20:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:56.425Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"64\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545985883
Jul 17 03:20:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:56.522Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_spec_paused\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"local-path-provisioner\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545993389
Jul 17 03:20:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:56.526Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_daemonset_status_desired_number_scheduled\", asset=\"pcap-tsgx03\", asset_id=\"73\", daemonset=\"dign-client\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545993389
Jul 17 03:20:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:56.534Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"shm\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/f8ed246a0799f0e1cd9cc46eb0e6fe6e684fb4f9ca3b265b7515b8205436505d/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545985883
Jul 17 03:20:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:56.571Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_node_status_allocatable\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", node=\"localhost\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", resource=\"memory\", service=\"prometheus-kube-state-metrics\", unit=\"byte\"}" timestamp=1689545993389
Jul 17 03:20:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:56.596Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"tmpfs\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/var/lib/kubelet/pods/5b5ea4ec-9b19-421d-88cb-9ffacec87f13/volumes/kubernetes.io~projected/kube-api-access-kl6pl\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545985883
Jul 17 03:20:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:56.607Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_discards_transmitted_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"mlx5_5\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545985883
Jul 17 03:20:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:56.611Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_drop_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens1f0np0\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545985883
Jul 17 03:20:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:56.640Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_carrier_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"vethaa604f97\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545985883
Jul 17 03:20:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:56.768Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_terminated\", asset=\"pcap-tsgx03\", asset_id=\"73\", container=\"prometheus-alertmanager-configmap-reload\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-alertmanager-6df7f457b6-7bgfr\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"799dcfe0-5d9a-4a69-9a0d-087901712118\"}" timestamp=1689545993389
Jul 17 03:20:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:57.089Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_dormant\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"eno35np2\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545985883
Jul 17 03:20:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:57.106Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-traefik-crd-blxb5\", project=\"TSG-OS\", project_id=\"7\", reason=\"Evicted\", service=\"prometheus-kube-state-metrics\", uid=\"fa6987b7-5e6f-40c1-a86c-e5d47b725265\"}" timestamp=1689545993389
Jul 17 03:20:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:57.141Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"86\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545985883
Jul 17 03:20:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:57.156Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_created\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"kubernetes-dashboard-csrf\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545993389
Jul 17 03:20:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:57.164Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"108\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545985883
Jul 17 03:20:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:57.197Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-journald.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"notify\", type=\"service\"}" timestamp=1689545983372
Jul 17 03:20:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:57.227Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_gc_duration_seconds\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", quantile=\"0\", service=\"prometheus-node-exporter\"}" timestamp=1689545985883
Jul 17 03:20:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:57.254Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_spec_replicas\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"ipmi-exporter-6f8464dd94\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545993389
Jul 17 03:20:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:57.267Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_up_changes_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth58e91056\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545985883
Jul 17 03:20:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:57.294Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_drop_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens6f1np1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545985883
Jul 17 03:20:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:57.315Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"38\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545985883
Jul 17 03:20:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:57.317Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cryptsetup.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"target\"}" timestamp=1689545983372
Jul 17 03:20:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:57.410Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"81\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545985883
Jul 17 03:20:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:57.429Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"rpc-statd-notify.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"service\"}" timestamp=1689545983372
Jul 17 03:20:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:57.452Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:20-0000:20:03.1-0000:21:00.1-net-ens2f1np1.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689545983372
Jul 17 03:20:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:57.456Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:00-0000:00:01.1-0000:01:00.0-net-ens1f0np0.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689545983372
Jul 17 03:20:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:57.645Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-virtual-net-vethd3018e52.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689545983372
Jul 17 03:20:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:57.784Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-f8ed246a0799f0e1cd9cc46eb0e6fe6e684fb4f9ca3b265b7515b8205436505d-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689545983372
Jul 17 03:20:58 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:58.454Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_sockstat_UDPLITE6_inuse\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"OLAP-Node-Exporter-pcap-dellr740-dt001\", endpoint_id=\"251\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689545997557
Jul 17 03:20:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:59.800Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_frees_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545985071
Jul 17 03:20:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:20:59.990Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_compressed_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth47e81471\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545989083
Jul 17 03:21:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:00.129Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"93\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545989083
Jul 17 03:21:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:00.400Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2did-scsi\\\\x2dSDELL_PERC_H345_Adp_009dff86e3be672b2c00ecd485f0a7ce\\\\x2dpart2.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689545984879
Jul 17 03:21:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:00.418Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"75\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545989083
Jul 17 03:21:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:00.425Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"slice\"}" timestamp=1689545984879
Jul 17 03:21:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:00.439Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_avail_bytes\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"shm\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/bc21478fdc5355bf15960d43a05074574abd6e1478c55132a193b1b21c029481/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545989083
Jul 17 03:21:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:00.448Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-virtual-net-vethcfad1fbd.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689545984879
Jul 17 03:21:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:00.512Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-virtio_dign_c.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689545984879
Jul 17 03:21:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:00.564Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_system_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", container=\"firewall\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689545988893
Jul 17 03:21:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:00.886Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_bytes_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens3f0np0\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545989083
Jul 17 03:21:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:00.909Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"coredns\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545987425
Jul 17 03:21:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:00.960Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_frame_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens1f1np1\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545989083
Jul 17 03:21:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:00.967Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"26\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545989083
Jul 17 03:21:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:00.989Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"77\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545989083
Jul 17 03:21:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:01.009Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_errs_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth55faa9d3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545989083
Jul 17 03:21:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:01.097Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx01\", asset_id=\"51\", container=\"prometheus-node-exporter\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Total\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545993195
Jul 17 03:21:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:01.143Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_avail_bytes\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"shm\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/6e01a4c6797c0850c82092afa712b52a2f8af6a46b4c4b4ae6c792284bef3e3a/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545989083
Jul 17 03:21:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:01.157Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_local_link_integrity_errors_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_10\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545989083
Jul 17 03:21:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:01.161Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_carrier_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens2f1np1\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545989083
Jul 17 03:21:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:01.194Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_errs_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"vethb97771d9\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545989083
Jul 17 03:21:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:01.267Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-random-seed.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689545984879
Jul 17 03:21:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:01.301Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx01\", asset_id=\"51\", container=\"configmap-reloader\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Read\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545985008
Jul 17 03:21:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:01.317Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_start_time_seconds\", asset=\"msh-tsgx01\", asset_id=\"51\", container=\"prometheus-node-exporter\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545994593
Jul 17 03:21:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:01.330Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_swap_limit_bytes\", asset=\"msh-tsgx01\", asset_id=\"51\", container=\"kube-multus\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545994593
Jul 17 03:21:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:01.348Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_files_free\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"shm\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/9ff1ff13dfcf7c52ab9d3875863fbbe2cc19fcde353d1f757adc5be88a7d818c/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545989083
Jul 17 03:21:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:01.361Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2dpath-pci\\\\x2d0000:81:00.0\\\\x2dscsi\\\\x2d0:2:0:0\\\\x2dpart1.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689545984879
Jul 17 03:21:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:01.378Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"77\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545989083
Jul 17 03:21:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:01.397Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_name_assign_type\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens4f1np1\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545989083
Jul 17 03:21:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:01.537Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_mapped_file\", asset=\"msh-tsgx01\", asset_id=\"51\", container=\"monit-stream\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545975766
Jul 17 03:21:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:01.607Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"msh-tsgx01\", asset_id=\"51\", condition=\"Progressing\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"systemd-exporter\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"false\"}" timestamp=1689545987425
Jul 17 03:21:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:01.620Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_owner\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", job_name=\"helm-install-traefik\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", owner_is_controller=\"<none>\", owner_kind=\"<none>\", owner_name=\"<none>\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545987425
Jul 17 03:21:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:01.690Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_labels\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_endpoint=\"mrzcpd-exporter\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545987425
Jul 17 03:21:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:01.708Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_annotations\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"ipmi-exporter\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689545987425
Jul 17 03:21:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:01.727Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"88\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689545989083
Jul 17 03:21:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:01.749Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_bytes_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"bond0\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545989083
Jul 17 03:21:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:02.440Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"msh-tsgx02\", asset_id=\"63\", cpu=\"9\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545998318
Jul 17 03:21:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:02.459Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kafka_server_socket_server_metrics_request_rate\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-Kafka-msh-dellr740-dt002\", endpoint_id=\"95\", listener=\"SASL_PLAINTEXT\", module=\"DC-Kafka\", module_id=\"5\", networkProcessor=\"0\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546002018
Jul 17 03:21:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:02.558Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failcnt\", asset=\"msh-tsgx02\", asset_id=\"63\", container=\"redis\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545985008
Jul 17 03:21:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:02.564Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"phy_rx_packets_total\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", dev=\"ens1f1np1\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"mrzcpd-exporter\"}" timestamp=1689545997995
Jul 17 03:21:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:02.580Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-eae093ead7efd40cc68023894f5e82bb1842e37f6341a5751d947c53f08358eb.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"scope\"}" timestamp=1689545988447
Jul 17 03:21:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:02.600Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"rsyslog.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"service\"}" timestamp=1689545988447
Jul 17 03:21:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:02.721Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_swap_limit_bytes\", asset=\"msh-tsgx02\", asset_id=\"63\", container=\"exporter\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545992003
Jul 17 03:21:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:03.200Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-8efce52b\\\\x2dd99a\\\\x2d4936\\\\x2db902\\\\x2dc7b03a21873d-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2drwh4p.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689545988447
Jul 17 03:21:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:03.244Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failcnt\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689545979244
Jul 17 03:21:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:03.305Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"net_conntrack_dialer_conn_failed_total\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", dialer_name=\"111\", instance=\"10.10.20.159:10090\", reason=\"refused\", type=\"global\"}" timestamp=1689546002989
Jul 17 03:21:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:03.329Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"msh-tsgx02\", asset_id=\"63\", condition=\"false\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"cm-policy-local-cache-redis-master-69967b5879-h2tzb\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"695dd1c7-6904-4ca3-aa3b-49efdba3b420\"}" timestamp=1689545991412
Jul 17 03:21:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:03.352Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"loadmodules.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"service\"}" timestamp=1689545988447
Jul 17 03:21:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:03.363Z caller=dedupe.go:112 component=remote level=info remote_name=52db6f url=http://10.10.10.159:10090/nz-agent/prometheus/proxy/api/v1/write msg="Dropped sample for series that was not explicitly dropped via relabelling" ref=4436650
Jul 17 03:21:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:03.399Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_state_started\", asset=\"msh-tsgx02\", asset_id=\"63\", container=\"kube-multus\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"kube-multus-ds-45cnz\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"7adc00b9-e646-4822-9ef0-1a15d34bda7e\"}" timestamp=1689545991412
Jul 17 03:21:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:03.422Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link_mode\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth421cdf56\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545998318
Jul 17 03:21:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:03.445Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"vethca16d75a\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545998318
Jul 17 03:21:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:03.475Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"prometheus_target_sync_length_seconds\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", instance=\"10.10.20.159:10090\", quantile=\"0.5\", scrape_job=\"113\", type=\"global\"}" timestamp=1689546002989
Jul 17 03:21:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:03.638Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"msh-tsgx02\", asset_id=\"63\", cpu=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689545998318
Jul 17 03:21:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:03.746Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_bytes_total\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", interface=\"ens4f0np0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545991232
Jul 17 03:21:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:03.777Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"msh-tsgx02\", asset_id=\"63\", container=\"ipmi-exporter\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"uninterruptible\"}" timestamp=1689545982340
Jul 17 03:21:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:04.353Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_io_time_weighted_seconds_total\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda2\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545991232
Jul 17 03:21:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:04.511Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"net_conntrack_dialer_conn_failed_total\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", dialer_name=\"311\", instance=\"10.10.20.159:10090\", reason=\"resolution\", type=\"global\"}" timestamp=1689546002989
Jul 17 03:21:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:04.581Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"prometheus_http_response_size_bytes_bucket\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", handler=\"/api/v1/query\", instance=\"10.10.20.159:10090\", le=\"1000\", type=\"global\"}" timestamp=1689546002989
Jul 17 03:21:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:04.958Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"net_conntrack_dialer_conn_failed_total\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", dialer_name=\"asset_ping_35\", instance=\"10.10.20.159:10090\", reason=\"resolution\", type=\"global\"}" timestamp=1689546002989
Jul 17 03:21:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:04.983Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"prometheus_target_sync_length_seconds\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", instance=\"10.10.20.159:10090\", quantile=\"0.01\", scrape_job=\"asset_ping_101\", type=\"global\"}" timestamp=1689546002989
Jul 17 03:21:05 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:05.117Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"net_conntrack_dialer_conn_failed_total\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", dialer_name=\"459\", instance=\"10.10.20.159:10090\", reason=\"resolution\", type=\"global\"}" timestamp=1689546002989
Jul 17 03:21:05 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:05.134Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"prometheus_tsdb_compactions_total\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", instance=\"10.10.20.159:10090\", type=\"global\"}" timestamp=1689546002989
Jul 17 03:21:05 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:05.147Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", cpu=\"16\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-dt001\", endpoint_id=\"225\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546002988
Jul 17 03:21:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:06.614Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_packets_total\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"cadvisor\", interface=\"ens2f0np0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546001119
Jul 17 03:21:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:06.639Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"twa-tsgx01\", asset_id=\"91\", container=\"telegraf\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", state=\"uninterruptible\"}" timestamp=1689545994244
Jul 17 03:21:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:06.717Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_net_dev_group\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"virtio_dign_c\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546003910
Jul 17 03:21:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:06.763Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_queue_length\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethdd29a3ce\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546003910
Jul 17 03:21:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:07.083Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_usage_seconds_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cpu=\"total\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546002930
Jul 17 03:21:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:07.131Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_packets_total\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"cadvisor\", interface=\"eth0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546002129
Jul 17 03:21:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:07.180Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-kernel-config.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689545993582
Jul 17 03:21:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:07.275Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Dup_Udp_Pkt\", app_name=\"sapp4\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", vsysId=\"1\"}" timestamp=1689546004123
Jul 17 03:21:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:07.356Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_metadata_resource_version\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.cm-policy-local-cache.v1\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546001227
Jul 17 03:21:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:07.398Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cpu=\"50\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546003910
Jul 17 03:21:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:07.474Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cpu=\"19\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546003910
Jul 17 03:21:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:07.476Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_working_set_bytes\", asset=\"twa-tsgx01\", asset_id=\"91\", container=\"kubernetes-dashboard\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689545997142
Jul 17 03:21:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:07.496Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cpu=\"26\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546003910
Jul 17 03:21:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:07.534Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cpu=\"92\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546003910
Jul 17 03:21:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:07.545Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db167\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689545998290
Jul 17 03:21:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:07.556Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"auditd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"forking\", type=\"service\"}" timestamp=1689545993582
Jul 17 03:21:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:11.395Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_STATS_metric_compactionInputFileCount_25th_percentile\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546008804
Jul 17 03:21:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:11.404Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_flushesQueuedCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", namespace=\"SYSTEM\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"ed92f2fb3f7db9aa4461f8283225dec7\", table=\"LOG\", type=\"data\"}" timestamp=1689546010147
Jul 17 03:21:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:11.452Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_PutBatch_98th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Server\"}" timestamp=1689546010147
Jul 17 03:21:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:11.515Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_putCount\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"9546c9406405c7b2c6e25082488d9256\", table=\"default\", type=\"data\"}" timestamp=1689546008804
Jul 17 03:21:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:11.545Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_maxFlushQueueSize\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"rtp_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"d2790ca158e989012e33b084e5563e8f\", table=\"index_time_rtp_hos_bucket\", type=\"index\"}" timestamp=1689546010147
Jul 17 03:21:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:11.561Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_compactionsCompletedCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"troubleshooting_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"606f43bb8ff18673fedf8b27c73d3dcd\", table=\"index_filename_troubleshooting_hos_bucket\", type=\"index\"}" timestamp=1689546010147
Jul 17 03:21:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:11.574Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_session_record_hos_bucket_metric_deleteTime_mean\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546010147
Jul 17 03:21:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:11.593Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_4_metric_flushOutputSize_99th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546010147
Jul 17 03:21:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:11.608Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_avgStoreFileAge\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"firewall_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"d70f3a5f4d655968d670685856c6115e\", table=\"index_partfile_firewall_hos_bucket\", type=\"index\"}" timestamp=1689546008804
Jul 17 03:21:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:11.630Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_compactionsFailedCount\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"9546c9406405c7b2c6e25082488d9256\", table=\"default\", type=\"data\"}" timestamp=1689546008804
Jul 17 03:21:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:11.675Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_maxStoreFileAge\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", namespace=\"SYSTEM\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"2f6b6350c5e74cb4f4b4cf9e7376455c\", table=\"CHILD_LINK\", type=\"data\"}" timestamp=1689546010147
Jul 17 03:21:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:11.682Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_CHILD_LINK_metric_scanTime_mean\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546010147
Jul 17 03:21:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:11.710Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_193_1_metric_majorCompactionOutputSize_99th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546010147
Jul 17 03:21:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:11.736Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_memStoreSize\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"troubleshooting_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"a3a56f381eea194ea2476e270c51204f\", table=\"index_partfile_troubleshooting_hos_bucket\", type=\"index\"}" timestamp=1689546010147
Jul 17 03:21:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:11.759Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_2_metric_majorCompactedInputBytes\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546010147
Jul 17 03:21:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:11.762Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_ScanTime_99th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Server\"}" timestamp=1689546010147
Jul 17 03:21:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:11.789Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_1_metric_compactionTime_max\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546010147
Jul 17 03:21:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:11.797Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_maxStoreFileAge\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"knowledge_base_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"d7545eddd6684b532e594f03edaf2b47\", table=\"index_time_knowledge_base_hos_bucket\", type=\"index\"}" timestamp=1689546010147
Jul 17 03:21:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:11.806Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_193_1_metric_getTime_25th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546010147
Jul 17 03:21:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:11.814Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_default_metric_incrementTime_99th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546010147
Jul 17 03:21:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:11.831Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_194_3_metric_majorCompactionInputSize_mean\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546010147
Jul 17 03:21:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:11.861Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_MUTEX_metric_incrementTime_99_9th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546010147
Jul 17 03:21:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:11.879Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_memStoreSize\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"assessment_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"e09d5f86b61240f1a3ed13b1100f1939\", table=\"index_filename_assessment_hos_bucket\", type=\"index\"}" timestamp=1689546010147
Jul 17 03:21:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:11.922Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_4_metric_deleteTime_99_9th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546010147
Jul 17 03:21:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:11.941Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_session_record_hos_bucket_metric_compactionInputSize_mean\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546010147
Jul 17 03:21:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:12.001Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_194_2_metric_flushTime_min\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546010147
Jul 17 03:21:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:12.012Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_AppendTime_95th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"WAL\"}" timestamp=1689546010147
Jul 17 03:21:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:12.045Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_tsg_table_report_result_metric_majorCompactionInputFileCount_max\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546010147
Jul 17 03:21:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:12.057Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_numBytesCompactedCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"proxy_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"acbd290c0645e422c9b3249c06c768ab\", table=\"index_filename_proxy_hos_bucket\", type=\"index\"}" timestamp=1689546010147
Jul 17 03:21:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:12.195Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_frequency_min_hertz\", asset=\"twa-dellr740-dt002\", asset_id=\"39\", cpu=\"33\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"NC-Gohangout-twa-dellr740-dt002\", endpoint_id=\"179\", module=\"DC-Gohangout\", module_id=\"35\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546011525
Jul 17 03:21:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:12.225Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_memory_DirectMap4k_bytes\", asset=\"twa-dellr740-dt002\", asset_id=\"39\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"OLAP-Node-Exporter-twa-dellr740-dt002\", endpoint_id=\"259\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546012057
Jul 17 03:21:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:12.322Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_info\", address=\"78:ac:44:24:36:42\", asset=\"twa-dellr740-dt002\", asset_id=\"39\", broadcast=\"ff:ff:ff:ff:ff:ff\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"em1\", duplex=\"full\", endpoint=\"OLAP-Node-Exporter-twa-dellr740-dt002\", endpoint_id=\"259\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", operstate=\"up\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546012057
Jul 17 03:21:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:12.387Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_MUTEX_metric_majorCompactionInputSize_90th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546010147
Jul 17 03:21:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:12.577Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_device_id\", asset=\"twa-dellr740-dt002\", asset_id=\"39\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"em1\", endpoint=\"OLAP-Node-Exporter-twa-dellr740-dt002\", endpoint_id=\"259\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546012057
Jul 17 03:21:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:12.650Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-dellr740-dt002\", asset_id=\"39\", cpu=\"1\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"OLAP-Node-Exporter-twa-dellr740-dt002\", endpoint_id=\"259\", mode=\"irq\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546012057
Jul 17 03:21:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:12.692Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-dellr740-dt002\", asset_id=\"39\", cpu=\"19\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"OLAP-Node-Exporter-twa-dellr740-dt002\", endpoint_id=\"259\", mode=\"user\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546012057
Jul 17 03:21:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:12.724Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_protocol_type\", asset=\"twa-dellr740-dt002\", asset_id=\"39\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethec37db3\", endpoint=\"OLAP-Node-Exporter-twa-dellr740-dt002\", endpoint_id=\"259\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546012057
Jul 17 03:21:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:12.863Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_4_metric_majorCompactionInputSize_98th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546010147
Jul 17 03:21:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:12.963Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_flags\", asset=\"twa-dellr740-dt002\", asset_id=\"39\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"em1\", endpoint=\"OLAP-Node-Exporter-twa-dellr740-dt002\", endpoint_id=\"259\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546012057
Jul 17 03:21:13 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:13.086Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_FUNCTION_metric_incrementTime_99th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546010147
Jul 17 03:21:13 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:13.105Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"java_lang_GarbageCollector_LastGcInfo_memoryUsageAfterGc_committed\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", key=\"Compressed Class Space\", module=\"DC-HBase\", module_id=\"3\", name=\"ParNew\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546010147
Jul 17 03:21:13 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:13.170Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_bytes_total\", asset=\"twa-dellr740-dt002\", asset_id=\"39\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"em3\", endpoint=\"OLAP-Node-Exporter-twa-dellr740-dt002\", endpoint_id=\"259\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546012057
Jul 17 03:21:13 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:13.860Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"twa-dellr740-dt002\", asset_id=\"39\", cpu=\"0\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"OLAP-Node-Exporter-twa-dellr740-dt002\", endpoint_id=\"259\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546012057
Jul 17 03:21:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:15.345Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_drop_total\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"bond0\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546001230
Jul 17 03:21:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:15.355Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"ext4\", name=\"var-lib-kubelet-pods-64082a69\\\\x2d3be2\\\\x2d4b5e\\\\x2d9e3a\\\\x2dc0da16576d9e-volume\\\\x2dsubpaths-config\\\\x2dvolume-telegraf\\\\x2dsecurity-1.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689546007046
Jul 17 03:21:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:15.362Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"90\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546001230
Jul 17 03:21:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:15.421Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx07\", asset_id=\"103\", container=\"dign-server-dns\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Discard\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546007641
Jul 17 03:21:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:15.512Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_last_seen\", asset=\"twa-tsgx07\", asset_id=\"103\", container=\"prometheus-server-configmap-reload\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546009185
Jul 17 03:21:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:15.650Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx07\", asset_id=\"103\", container=\"kube-multus\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Read\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546007472
Jul 17 03:21:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:15.699Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_metadata_resource_version\", asset=\"twa-tsgx07\", asset_id=\"103\", configmap=\"prometheus-alertmanager\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546008076
Jul 17 03:21:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:15.713Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"nsfs\", name=\"run-netns-cni\\\\x2db75bceb6\\\\x2dfd63\\\\x2df153\\\\x2dce5c\\\\x2dac23e0ed0a58.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689546007046
Jul 17 03:21:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:15.728Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-6558180a2997c416bd962435cc8f217683e4cfff330c39e1d6835731ad64560f.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"scope\"}" timestamp=1689546007046
Jul 17 03:21:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:15.736Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"twa-tsgx07\", asset_id=\"103\", container=\"prometheus-alertmanager-configmap-reload\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"sleeping\"}" timestamp=1689546006240
Jul 17 03:21:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:15.773Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_node_status_condition\", asset=\"twa-tsgx07\", asset_id=\"103\", condition=\"MemoryPressure\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", node=\"localhost\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"false\"}" timestamp=1689546008076
Jul 17 03:21:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:15.788Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_labels\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-0-firewall-7c78fcbddc\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546008076
Jul 17 03:21:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:15.834Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"70\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546001230
Jul 17 03:21:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:15.930Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"126\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546001230
Jul 17 03:21:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:16.072Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_packets_transmitted_total\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_4\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546001230
Jul 17 03:21:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:16.121Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Ipv6_Pkt\", app_name=\"sapp4\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", vsysId=\"1\"}" timestamp=1689546011690
Jul 17 03:21:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:16.348Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx07\", asset_id=\"103\", container=\"dign-client\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Async\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546007355
Jul 17 03:21:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:16.373Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_info\", asset=\"twa-tsgx07\", asset_id=\"103\", created_by_kind=\"ReplicaSet\", created_by_name=\"prometheus-pushgateway-bd476bcf9\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", host_ip=\"10.10.30.167\", host_network=\"false\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", node=\"localhost\", nz_agent_id=\"3\", pod=\"prometheus-pushgateway-bd476bcf9-d7fgf\", pod_ip=\"10.42.0.66\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"eda00de6-45a4-4eaf-8b0e-75c0b9c1b061\"}" timestamp=1689546008076
Jul 17 03:21:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:16.537Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_multicast_total\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethe08168cb\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546001230
Jul 17 03:21:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:16.550Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_spec_paused\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"cm-policy-local-cache-redis-master\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546008076
Jul 17 03:21:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:16.736Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_annotations\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-server-6ff784d758-sgqd4\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"0ed53af9-6e57-4bfe-bb64-b0aa4edce9c5\"}" timestamp=1689546008076
Jul 17 03:21:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:16.756Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled_time\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-node-exporter-pfqsj\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"85ce1d65-33c2-41b7-81ab-b25e87a182c7\"}" timestamp=1689546008076
Jul 17 03:21:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:16.924Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_ports\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_endpoint=\"kube-dns\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", port_name=\"metrics\", port_number=\"9153\", port_protocol=\"TCP\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546008076
Jul 17 03:21:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:16.978Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"10\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546001230
Jul 17 03:21:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:19.095Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"clickhouse_arena_alloc_chunks_total\", asset=\"twa-dellr740-dt003\", asset_id=\"41\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"NC-Clickhouse-twa-dellr740-dt003\", endpoint_id=\"91\", module=\"NC-Clickhouse\", module_id=\"29\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546018301
Jul 17 03:21:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:19.825Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"5\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546017612
Jul 17 03:21:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:19.869Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_address_assign_type\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"virtio_dign_s\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546017612
Jul 17 03:21:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:19.912Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"45\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546017612
Jul 17 03:21:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:19.994Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_net_dev_group\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"virtio_dign_c\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546017612
Jul 17 03:21:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:20.004Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_sockets\", asset=\"twa-tsgx06\", asset_id=\"101\", container=\"telegraf-security\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689546004012
Jul 17 03:21:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:20.016Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Sync\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689546007160
Jul 17 03:21:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:20.052Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"75\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546017612
Jul 17 03:21:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:20.069Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_observed_generation\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"cm-policy-local-cache-redis-master-69967b5879\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546013949
Jul 17 03:21:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:20.093Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"93\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546017612
Jul 17 03:21:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:20.116Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_net_dev_group\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens3f0np0\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546017612
Jul 17 03:21:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:20.134Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dracut-initqueue.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"service\"}" timestamp=1689546004356
Jul 17 03:21:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:20.138Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-sda.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689546004356
Jul 17 03:21:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:20.157Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_queue_length\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethef8c731a\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546017612
Jul 17 03:21:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:20.161Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_bytes_total\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"cadvisor\", interface=\"ens6f0np0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546009844
Jul 17 03:21:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:20.182Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"65\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546017612
Jul 17 03:21:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:20.353Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"105\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546008263
Jul 17 03:21:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:20.371Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas_updated\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"prometheus-server\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546013828
Jul 17 03:21:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:20.410Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_init_container_resource_requests\", asset=\"pcap-tsgx10\", asset_id=\"89\", container=\"install-multus-binary\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", node=\"localhost\", nz_agent_id=\"3\", pod=\"kube-multus-ds-sqrb7\", project=\"TSG-OS\", project_id=\"7\", resource=\"memory\", service=\"prometheus-kube-state-metrics\", uid=\"ee8afe42-b7fb-4840-a9ee-3d0e7212c356\", unit=\"byte\"}" timestamp=1689546013828
Jul 17 03:21:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:20.441Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_labels\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.tsg-traffic-engine-numa-0.v4\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546013828
Jul 17 03:21:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:20.442Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-f77a813ab9391ef607b1328371c74b32ad3ce56149b7b03a3b87ee35130a3ac1.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"scope\"}" timestamp=1689546010657
Jul 17 03:21:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:20.467Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-d45e136aa511981e1cddf3b122cea374a162c45d8a566542d022c884d7d97985-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689546004356
Jul 17 03:21:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:20.484Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"var-lib-kubelet-pods-c8175cdd\\\\x2d27fb\\\\x2d4976\\\\x2da041\\\\x2dad5b277f4e45-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2djvgwx.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689546010657
Jul 17 03:21:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:20.485Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2did-scsi\\\\x2d362cea7f0add598002c2c8d3154fff241\\\\x2dpart3.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689546010657
Jul 17 03:21:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:20.519Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db196\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546011998
Jul 17 03:21:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:20.534Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"112\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546017612
Jul 17 03:21:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:20.588Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"NetworkManager-wait-online.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689546010657
Jul 17 03:21:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:20.594Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_daemonset_status_current_number_scheduled\", asset=\"twa-tsgx06\", asset_id=\"101\", daemonset=\"dign-client\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546013949
Jul 17 03:21:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:20.612Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_io_current\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/shm\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546008446
Jul 17 03:21:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:20.615Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", container=\"kube-state-metrics\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"cadvisor\", failure_type=\"pgfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"hierarchy\"}" timestamp=1689546010281
Jul 17 03:21:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:20.635Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-9fbd3ff04dd48db4b39f55e3c9158c39d3277e82aad9856f80daed4ca7246221-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689546010657
Jul 17 03:21:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:20.640Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_id\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth946c4097\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546017612
Jul 17 03:21:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:20.665Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"63\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546017612
Jul 17 03:21:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:20.706Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Succeeded\", pod=\"helm-install-traefik-crd-6hp8h\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"e4381804-2831-4c31-a46e-78f802625973\"}" timestamp=1689546013949
Jul 17 03:21:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:20.713Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_cpu_shares\", asset=\"pcap-tsgx10\", asset_id=\"89\", container=\"prometheus-alertmanager-configmap-reload\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546015735
Jul 17 03:21:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:20.725Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"72\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546017612
Jul 17 03:21:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:20.734Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_info\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_endpoint=\"dashboard-metrics-scraper\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546013828
Jul 17 03:21:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:20.756Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_netstat_TcpExt_TCPTimeouts\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546008263
Jul 17 03:21:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:20.777Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"emergency.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"target\"}" timestamp=1689546004356
Jul 17 03:21:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:20.910Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_packets_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth0aa05a07\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546008263
Jul 17 03:21:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:20.928Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"iscsi.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689546010657
Jul 17 03:21:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:20.972Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"82\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546008263
Jul 17 03:21:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:21.021Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_commands_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cmd=\"client\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546014840
Jul 17 03:21:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:21.041Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-95bc4fcc9e5e23097a950e3b31a5e3c8a263a6882a0f9b5ed6de93811d371f00.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"scope\"}" timestamp=1689546010657
Jul 17 03:21:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:21.069Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_hwmon_sensor_label\", asset=\"twa-tsgx06\", asset_id=\"101\", chip=\"pci0000:00_0000:00:18_3\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", label=\"tccd8\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", sensor=\"temp10\", service=\"prometheus-node-exporter\"}" timestamp=1689546017612
Jul 17 03:21:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:21.121Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-da67439d\\\\x2d61e0\\\\x2d42b8\\\\x2d80b6\\\\x2d55a738bae0d9-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2dtpjd8.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689546004356
Jul 17 03:21:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:21.149Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-virtual-net-veth9421f8b7.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689546004356
Jul 17 03:21:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:21.821Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_disk_read_time_seconds_total\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"sda\", endpoint=\"OLAP-Node-Exporter-twa-dellr740-dt001\", endpoint_id=\"257\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546021405
Jul 17 03:21:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:21.844Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"pcap-tsgx10\", asset_id=\"89\", condition=\"Available\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"systemd-exporter\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"false\"}" timestamp=1689546013828
Jul 17 03:21:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:21.869Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", cpu=\"10\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"OLAP-Node-Exporter-twa-dellr740-dt001\", endpoint_id=\"257\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546021405
Jul 17 03:21:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:21.981Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", cpu=\"29\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"OLAP-Node-Exporter-twa-dellr740-dt001\", endpoint_id=\"257\", mode=\"irq\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546021405
Jul 17 03:21:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:22.133Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", cpu=\"3\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"OLAP-Node-Exporter-twa-dellr740-dt001\", endpoint_id=\"257\", mode=\"user\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546021405
Jul 17 03:21:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:23.814Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2did-scsi\\\\x2dSDELL_PERC_H345_Adp_00c4cde91457202c2c00638284f0a7ce\\\\x2dpart5.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689546018416
Jul 17 03:21:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:23.843Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_metadata_generation\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"ipmi-exporter-6f8464dd94\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546020068
Jul 17 03:21:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:23.860Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"msh-tsgx09\", asset_id=\"55\", cpu=\"17\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546014790
Jul 17 03:21:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:24.165Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-852b631a3baff6085e592e514448bfb1292ec41588c97b9cc1e9c05320126a83-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689546018416
Jul 17 03:21:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:24.228Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-tmpfiles-clean.timer\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"timer\"}" timestamp=1689546018416
Jul 17 03:21:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:24.263Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_size_bytes\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda5\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", fstype=\"ext4\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/home\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546020160
Jul 17 03:21:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:24.322Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"metrics-server\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", failure_type=\"pgmajfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"hierarchy\"}" timestamp=1689546008953
Jul 17 03:21:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:24.344Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-tsg-traffic-engine-numa-0-n2fbk\", project=\"TSG-OS\", project_id=\"7\", reason=\"NodeAffinity\", service=\"prometheus-kube-state-metrics\", uid=\"044ab283-1226-4087-a5d1-3cb614606766\"}" timestamp=1689546020068
Jul 17 03:21:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:24.360Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_state_id\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_8\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546014790
Jul 17 03:21:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:24.361Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_cpu_period\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689546023419
Jul 17 03:21:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:24.371Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx09\", asset_id=\"55\", container=\"exporter\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Write\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546013393
Jul 17 03:21:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:24.386Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_rss\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689546012905
Jul 17 03:21:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:24.509Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"msh-tsgx09\", asset_id=\"55\", container=\"monit-stream\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", failure_type=\"pgfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"container\"}" timestamp=1689546009086
Jul 17 03:21:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:24.533Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_hwmon_chip_names\", asset=\"msh-tsgx09\", asset_id=\"55\", chip=\"0000:60:03_1_0000:63:00_3\", chip_name=\"bnxt_en\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546014790
Jul 17 03:21:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:24.553Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_commands_failed_calls_total\", asset=\"msh-tsgx09\", asset_id=\"55\", cmd=\"set\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546017019
Jul 17 03:21:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:24.560Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_labels\", asset=\"pcap-tsgx12\", asset_id=\"117\", configmap=\"shaping-tsg-traffic-engine-numa-1\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546011122
Jul 17 03:21:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:24.570Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-50568b47de6d5e587f8512a9f733c7bb7d4f4e270a38f018a817f2b012648f1d-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689546018416
Jul 17 03:21:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:24.579Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"tmpfs\", name=\"run-k3s-containerd-io.containerd.grpc.v1.cri-sandboxes-466358068c20fde9aa501a7288c283e62c6b11dd9e66f3ceed2e51731f8f4404-shm.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689546011102
Jul 17 03:21:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:24.602Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_info\", asset=\"msh-tsgx09\", asset_id=\"55\", container=\"traefik\", container_id=\"containerd://053f0c56911bcc9c7f6d53c1d3182f6651ff6416d7dbb9b114c6ff27a5254a87\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", image=\"docker.io/rancher/mirrored-library-traefik:2.9.10\", image_id=\"sha256:d1e26b5f8193d713eb8f39d0ab0e3d513278f86e41644b07e189e1aef536b335\", image_spec=\"rancher/mirrored-library-traefik:2.9.10\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"traefik-64f55bb67d-xmdbm\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"012398ff-fe58-4ee4-8731-8e61bf317bbf\"}" timestamp=1689546020068
Jul 17 03:21:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:24.618Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_data_transmitted_bytes_total\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546014790
Jul 17 03:21:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:24.632Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_packets_total\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens4f0np0\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546014790
Jul 17 03:21:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:24.638Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_labels\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", job_name=\"helm-install-configmap-reloader\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546020068
Jul 17 03:21:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:24.649Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"mrzcpd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"notify\", type=\"service\"}" timestamp=1689546011102
Jul 17 03:21:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:24.653Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_cpu_seconds_total\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-logind.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689546018416
Jul 17 03:21:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:24.678Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"dign-client\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"stopped\"}" timestamp=1689546011062
Jul 17 03:21:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:24.694Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"overlay\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-c94d881a81c90bd8c5090dd1e723684e6167d23e52727ce476255343d7ffc466-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689546018416
Jul 17 03:21:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:24.698Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_labels\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_endpoint=\"prometheus-alertmanager\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546011122
Jul 17 03:21:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:24.743Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_id\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth168605c4\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546020160
Jul 17 03:21:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:24.761Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db254\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546021795
Jul 17 03:21:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:24.807Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_spec_replicas\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"kubernetes-dashboard\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546011122
Jul 17 03:21:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:24.821Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"pcap-tsgx12\", asset_id=\"117\", condition=\"Progressing\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"prometheus-server\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"unknown\"}" timestamp=1689546011122
Jul 17 03:21:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:24.838Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_info\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_endpoint=\"kubernetes\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546011122
Jul 17 03:21:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:24.882Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-c6698dc9b63b39a0924a2de4a5b1b8dba506c36162dc30c362acc6e475551e25.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"scope\"}" timestamp=1689546018416
Jul 17 03:21:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:24.891Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_last_terminated_reason\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"monit-stream\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"mrzcpd-exporter-5cf7f967cc-rzgpz\", project=\"TSG-OS\", project_id=\"7\", reason=\"Unknown\", service=\"prometheus-kube-state-metrics\", uid=\"5724459f-9a6b-46d1-9437-b9002b60d530\"}" timestamp=1689546011122
Jul 17 03:21:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:24.906Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", failure_type=\"pgfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"hierarchy\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689546021280
Jul 17 03:21:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:24.923Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-burstable-podbd33971b_afbd_4dbb_935a_ec9c5b0f6a92.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"slice\"}" timestamp=1689546018416
Jul 17 03:21:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:24.928Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2dpath-pci\\\\x2d0000:81:00.0\\\\x2dscsi\\\\x2d0:2:0:0\\\\x2dpart2.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689546011102
Jul 17 03:21:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:24.966Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_labels\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"prometheus-alertmanager-6df7f457b6\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546011122
Jul 17 03:21:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:24.981Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-1756e142915abfc37edad68ac93fc2458bd0f45fbe879b4838e307d81b928d57.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"scope\"}" timestamp=1689546011102
Jul 17 03:21:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:25.018Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_name_assign_type\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"vethd1369752\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546020160
Jul 17 03:21:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:25.071Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_id\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth9642bab9\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546020160
Jul 17 03:21:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:25.115Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_fifo_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"vethc96a0a69\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546020160
Jul 17 03:21:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:25.131Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-ask-password-wall.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689546011102
Jul 17 03:21:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:25.165Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_namespace_annotations\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"consul\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546011122
Jul 17 03:21:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:25.178Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"14\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546020160
Jul 17 03:21:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:25.263Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_start_time_seconds\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"prometheus-alertmanager-configmap-reload\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546023419
Jul 17 03:21:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:25.338Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"125\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546020160
Jul 17 03:21:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:25.392Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_id\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"eno2\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546020160
Jul 17 03:21:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:25.988Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_max_fds\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-logind.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689546013907
Jul 17 03:21:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:26.382Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"106\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546015810
Jul 17 03:21:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:26.426Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_errs_total\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens6f0np0\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546015810
Jul 17 03:21:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:26.742Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_disk_discards_completed_total\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"sda\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546015810
Jul 17 03:21:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:26.784Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", phase=\"Failed\", pod=\"prometheus-kube-state-metrics-6b768c7bf8-c5glh\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"1d1df77f-66ac-4156-8ea3-66e8a287970d\"}" timestamp=1689546018744
Jul 17 03:21:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:26.892Z caller=dedupe.go:112 component=remote level=error remote_name=52db6f url=http://10.10.10.159:10090/nz-agent/prometheus/proxy/api/v1/write msg="non-recoverable error" count=1000 exemplarCount=0 err="server returned HTTP status 400 : out of order sample"
Jul 17 03:21:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:27.005Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-6dfb472b5c6f4e10ab26aea947c493f86c4b6a1735b0b3576b03366de4b17cd5.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"scope\"}" timestamp=1689546013907
Jul 17 03:21:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:27.315Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_created\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-traefik-crd-h8g7g\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"e303f019-5869-4dc2-afe6-c1107cceb13d\"}" timestamp=1689546018744
Jul 17 03:21:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:27.411Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_unicast_packets_received_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_6\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546022882
Jul 17 03:21:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:27.469Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_address_available\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_endpoint=\"shaping-announce-port-tsg-traffic-engine-numa-0\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546015682
Jul 17 03:21:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:27.564Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link_mode\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth1e7f8cb3\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546015810
Jul 17 03:21:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:27.836Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_errs_total\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens1f1np1\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546015810
Jul 17 03:21:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:27.842Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"126\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546022882
Jul 17 03:21:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:27.861Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_compressed_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens2f0np0\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546022882
Jul 17 03:21:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:28.172Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_info\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", job_name=\"helm-install-tsg-traffic-engine-numa-0\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546015682
Jul 17 03:21:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:28.238Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_info\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"prometheus-server\", container_id=\"containerd://188e2a0050f0b11a8d9261276524431439c6dc5158ed49c0c123d258fb2fd886\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", image=\"quay.io/prometheus/prometheus:v2.39.1\", image_id=\"sha256:6b9895947e9e4b2db6b271f999b3637e660185ae331021c3d47e6e6345a1c472\", image_spec=\"quay.io/prometheus/prometheus:v2.39.1\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-server-6ff784d758-bvjp8\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"0bdb24ec-e0f8-44e3-8d3f-a437820bc4e9\"}" timestamp=1689546015682
Jul 17 03:21:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:28.240Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"90\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546022882
Jul 17 03:21:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:28.371Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Unknown_Bit\", app_name=\"sapp4\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", vsysId=\"1\"}" timestamp=1689546026715
Jul 17 03:21:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:28.372Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db211\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546019446
Jul 17 03:21:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:31.294Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"45\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546020939
Jul 17 03:21:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:31.340Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_down_changes_total\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth1d757172\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546020939
Jul 17 03:21:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:31.800Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"55\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546019093
Jul 17 03:21:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:31.872Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"msh-tsgx05\", asset_id=\"67\", container=\"prometheus-alertmanager\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"cadvisor\", failure_type=\"pgmajfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"hierarchy\"}" timestamp=1689546005707
Jul 17 03:21:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:31.894Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"45\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546020791
Jul 17 03:21:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:31.919Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_rss\", asset=\"msh-tsgx05\", asset_id=\"67\", container=\"telegraf-security\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689546005546
Jul 17 03:21:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:32.177Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db217\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546016711
Jul 17 03:21:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:32.185Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"17\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546020791
Jul 17 03:21:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:32.196Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda4\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", fstype=\"ext4\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/var/lib/kubelet/pods/b3068a5c-f8e2-4c2d-89be-7648e87ab2c1/volume-subpaths/config-path/dign-server-dns/1\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546019093
Jul 17 03:21:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:32.223Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db246\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546026438
Jul 17 03:21:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:32.246Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"redis\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"sleeping\"}" timestamp=1689546017023
Jul 17 03:21:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:32.310Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"11\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546020791
Jul 17 03:21:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:32.437Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"twa-tsgx03\", asset_id=\"95\", container=\"kube-multus\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"sleeping\"}" timestamp=1689546013993
Jul 17 03:21:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:32.467Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"twa-tsgx03\", asset_id=\"95\", condition=\"Available\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"traefik\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"true\"}" timestamp=1689546030436
Jul 17 03:21:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:32.575Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"68\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546020605
Jul 17 03:21:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:32.655Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"63\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546020605
Jul 17 03:21:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:32.902Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db139\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546029260
Jul 17 03:21:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:33.230Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_complete\", asset=\"twa-tsgx05\", asset_id=\"99\", condition=\"false\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", job_name=\"helm-install-traefik\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546019894
Jul 17 03:21:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:33.237Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx05\", asset_id=\"99\", container=\"dign-client\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Discard\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546017071
Jul 17 03:21:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:33.359Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"44\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546020605
Jul 17 03:21:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:33.415Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_packets_total\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens2f0np0\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546020791
Jul 17 03:21:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:33.559Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_bytes_total\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens1f1np1\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546020605
Jul 17 03:21:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:33.668Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_swap\", asset=\"msh-tsgx05\", asset_id=\"67\", container=\"exporter\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546012008
Jul 17 03:21:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:33.690Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"scrape_samples_scraped\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_instance=\"10.42.0.63:9290\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"ipmi-exporter\"}" timestamp=1689546023896
Jul 17 03:21:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:33.695Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2dpath-pci\\\\x2d0000:81:00.0\\\\x2dscsi\\\\x2d0:2:0:0\\\\x2dpart5.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689546021015
Jul 17 03:21:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:33.709Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"microcode.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"service\"}" timestamp=1689546027691
Jul 17 03:21:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:33.738Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-51c8f219e5b27a33e124b9a570d092b90a047dd6a00d9b42a62a6700520d7680-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689546021015
Jul 17 03:21:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:33.743Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"tmpfs\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/data/var/lib/kubelet/pods/b3068a5c-f8e2-4c2d-89be-7648e87ab2c1/volumes/kubernetes.io~projected/kube-api-access-dtfzs\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546019093
Jul 17 03:21:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:33.796Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_last_seen\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689546015866
Jul 17 03:21:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:33.898Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_frame_total\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethfd281763\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546019093
Jul 17 03:21:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:33.987Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_packets_total\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"cadvisor\", interface=\"eno1\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546014924
Jul 17 03:21:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:34.001Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_created\", asset=\"msh-tsgx05\", asset_id=\"67\", configmap=\"chart-content-tsg-traffic-engine-numa-0\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546024068
Jul 17 03:21:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:34.002Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_daemonset_labels\", asset=\"twa-tsgx05\", asset_id=\"99\", daemonset=\"dign-server\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546019894
Jul 17 03:21:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:34.002Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-99e7b7f0\\\\x2d307f\\\\x2d4274\\\\x2da670\\\\x2dfaf429f178f7-volume\\\\x2dsubpaths-config\\\\x2dvolume-telegraf\\\\x2dsecurity-1.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689546021015
Jul 17 03:21:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:34.005Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_packets_total\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"cadvisor\", interface=\"ens6f0np0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546014624
Jul 17 03:21:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:34.020Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_address_assign_type\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"virtio_dign_c\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546020939
Jul 17 03:21:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:34.035Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dracut-pre-trigger.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689546027691
Jul 17 03:21:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:34.046Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546019093
Jul 17 03:21:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:34.062Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"fcoe.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689546027691
Jul 17 03:21:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:34.079Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"configmap-reloader\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Async\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546017418
Jul 17 03:21:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:34.165Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"1\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546020791
Jul 17 03:21:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:34.186Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_swap_limit_bytes\", asset=\"twa-tsgx05\", asset_id=\"99\", container=\"firewall\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689546018535
Jul 17 03:21:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:34.213Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_labels\", asset=\"twa-tsgx05\", asset_id=\"99\", configmap=\"prometheus-server\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546019894
Jul 17 03:21:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:34.241Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_client_recent_max_output_buffer_bytes\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546030141
Jul 17 03:21:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:34.355Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_errors_received_total\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_2\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546019093
Jul 17 03:21:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:34.359Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"26\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546020605
Jul 17 03:21:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:34.403Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"40\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546019093
Jul 17 03:21:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:34.461Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_reads_total\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"dign-server-dns\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546005032
Jul 17 03:21:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:34.548Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546019093
Jul 17 03:21:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:34.548Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_status_succeeded\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", job_name=\"helm-install-tsg-traffic-engine-numa-0\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546019894
Jul 17 03:21:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:40.109Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_commands_duration_seconds_total\", asset=\"msh-kvm-cm001\", asset_id=\"3\", cmd=\"flushdb\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"CM-Branch-Redis-msh-kvm-cm001\", endpoint_id=\"319\", host=\"192.168.20.157\", module=\"CM-Branch-Redis\", module_id=\"73\", nz_agent_id=\"3\", parent_asset=\"msh-dellr740-pve001\", parent_asset_id=\"1\", project=\"Central-Management\", project_id=\"5\"}" timestamp=1689546038583
Jul 17 03:21:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:40.597Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx04\", asset_id=\"97\", container=\"firewall\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Discard\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689546018135
Jul 17 03:21:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:40.607Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_restart_policy\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-alertmanager-6df7f457b6-vphf9\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", type=\"Always\", uid=\"94a4f272-a146-4944-832c-92f8c3e47728\"}" timestamp=1689546040034
Jul 17 03:21:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:42.419Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_info\", asset=\"pcap-tsgx01\", asset_id=\"71\", created_by_kind=\"DaemonSet\", created_by_name=\"svclb-traefik-7eb8511e\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", host_ip=\"10.10.10.161\", host_network=\"false\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", node=\"localhost\", nz_agent_id=\"3\", pod=\"svclb-traefik-7eb8511e-q59w6\", pod_ip=\"10.42.0.64\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"0a82591e-01f6-43ee-9af0-f8cc0036ad27\"}" timestamp=1689546035154
Jul 17 03:21:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:42.483Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_writes_total\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689546014363
Jul 17 03:21:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:42.575Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_created\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"coredns-64587b7d8\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546040034
Jul 17 03:21:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:42.607Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_swap_limit_bytes\", asset=\"pcap-tsgx01\", asset_id=\"71\", container=\"prometheus-alertmanager\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546040951
Jul 17 03:21:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:42.623Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"44\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546031647
Jul 17 03:21:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:42.626Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:20-0000:20:03.1-0000:21:00.1-net-ens2f1np1.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689546035764
Jul 17 03:21:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:42.677Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"88\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546031647
Jul 17 03:21:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:42.919Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_metadata_generation\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-1-firewall-786576fb8\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546035154
Jul 17 03:21:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:42.938Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_unicast_packets_received_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"mlx5_3\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546036242
Jul 17 03:21:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:43.004Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-eno34np1.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689546040671
Jul 17 03:21:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:43.099Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_reads_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", container=\"kube-multus\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546032989
Jul 17 03:21:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:43.111Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-prometheus-vzwtx\", project=\"TSG-OS\", project_id=\"7\", reason=\"Evicted\", service=\"prometheus-kube-state-metrics\", uid=\"1187ac6c-e3c7-4185-8a91-8bb4228b3b51\"}" timestamp=1689546040034
Jul 17 03:21:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:43.137Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"0\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546031647
Jul 17 03:21:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:43.164Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-node-exporter-xrmbl\", project=\"TSG-OS\", project_id=\"7\", reason=\"Shutdown\", service=\"prometheus-kube-state-metrics\", uid=\"9f402370-116a-4e69-9fa9-b0192f6cd502\"}" timestamp=1689546040034
Jul 17 03:21:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:43.188Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_multicast_total\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"eno36np3\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546031647
Jul 17 03:21:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:43.193Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_packets_dropped_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"cadvisor\", interface=\"ens8f0np0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546035057
Jul 17 03:21:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:43.247Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", cpu=\"1\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546036242
Jul 17 03:21:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:43.280Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", cpu=\"41\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546036242
Jul 17 03:21:45 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:45.500Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-5b7cc14f\\\\x2deef4\\\\x2d4809\\\\x2da696\\\\x2dcfb75e06120f-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2d4v7rb.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689546040671
Jul 17 03:21:45 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:45.513Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", container=\"dign-client\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"sleeping\"}" timestamp=1689546025936
Jul 17 03:21:45 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:45.590Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_resource_requests\", asset=\"pcap-tsgx01\", asset_id=\"71\", container=\"coredns\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", node=\"localhost\", nz_agent_id=\"3\", pod=\"coredns-79c6ccb74b-rpzl7\", project=\"TSG-OS\", project_id=\"7\", resource=\"memory\", service=\"prometheus-kube-state-metrics\", uid=\"be245aed-e373-405f-8d79-93c5764d578b\", unit=\"byte\"}" timestamp=1689546035154
Jul 17 03:21:48 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:48.149Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"clickhouse_block_discard_bytes_sdb\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Clickhouse-msh-dellr740-dt001\", endpoint_id=\"79\", module=\"NC-Clickhouse\", module_id=\"29\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546047155
Jul 17 03:21:48 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:48.789Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_193_1_metric_incrementTime_90th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546047438
Jul 17 03:21:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:49.312Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_194_2_metric_splitSuccessCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546047438
Jul 17 03:21:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:49.346Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_replicaid\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"assessment_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"43c96dd21aa1238d6db7226daea0110e\", table=\"index_filename_assessment_hos_bucket\", type=\"index\"}" timestamp=1689546047438
Jul 17 03:21:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:49.567Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_annotations\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-kube-state-metrics-6b768c7bf8-n4ck4\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"7be94d73-b8c6-4b83-adb9-10dac35bf41e\"}" timestamp=1689546039668
Jul 17 03:21:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:49.643Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_3_metric_deleteBatchTime_99_9th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546047438
Jul 17 03:21:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:49.644Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_observed_generation\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-0-firewall-5588f7fcbc\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546039668
Jul 17 03:21:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:49.698Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx10\", asset_id=\"109\", cpu=\"94\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546041797
Jul 17 03:21:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:49.744Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"auth-rpcgss-module.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"service\"}" timestamp=1689546041057
Jul 17 03:21:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:49.756Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_errors_total\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", err=\"ERR\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546046625
Jul 17 03:21:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:49.762Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_194_1_metric_flushOutputSize_99th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546047438
Jul 17 03:21:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:49.797Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-ask-password-console.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"simple\", type=\"service\"}" timestamp=1689546041057
Jul 17 03:21:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:50.016Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_PauseTimeWithGc_median\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Server\"}" timestamp=1689546047438
Jul 17 03:21:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:50.155Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_io_time_weighted_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"overlay_0-177\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546039361
Jul 17 03:21:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:50.630Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_created\", asset=\"twa-tsgx10\", asset_id=\"109\", configmap=\"chart-content-traefik-crd\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546039668
Jul 17 03:21:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:50.926Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_xfs_inode_operation_missed_total\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"sda2\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-olap003\", endpoint_id=\"235\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"yarn_resourcemanager,zookeeper,nacos,kafka,spark,druid_data,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546049714
Jul 17 03:21:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:50.957Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db226\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546046625
Jul 17 03:21:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:51.078Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db48\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546046625
Jul 17 03:21:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:51.207Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"clevis-luks-askpass.path\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"path\"}" timestamp=1689546041057
Jul 17 03:21:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:51.270Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_labels\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.tsg-traffic-engine-numa-0.v2\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546039668
Jul 17 03:21:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:51.274Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"100\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546046709
Jul 17 03:21:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:51.701Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_memory_Writeback_bytes\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546046709
Jul 17 03:21:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:51.719Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"firewall\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", state=\"sleeping\"}" timestamp=1689546039013
Jul 17 03:21:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:51.754Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"system-mlnx_interface_mgr.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"slice\"}" timestamp=1689546041057
Jul 17 03:21:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:51.805Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"twa-tsgx08\", asset_id=\"105\", condition=\"false\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"svclb-traefik-8650ed90-pfkvd\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"b1413324-fd7a-4809-abab-764ba720b05d\"}" timestamp=1689546041568
Jul 17 03:21:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:51.812Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_bytes_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"eno34np1\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546046709
Jul 17 03:21:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:51.872Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"twa-tsgx10\", asset_id=\"109\", condition=\"true\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"dign-client-qd7p8\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"1b9e7e11-9912-45e0-831b-8143d0877131\"}" timestamp=1689546039668
Jul 17 03:21:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:51.917Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-1c6adf295e169e5435abc78195bc6d7e760d8521681d770d7d0e756a96e2a04e.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"scope\"}" timestamp=1689546041057
Jul 17 03:21:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:52.099Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db154\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546047556
Jul 17 03:21:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:52.286Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-kube-state-metrics-6b768c7bf8-rkth4\", project=\"TSG-OS\", project_id=\"7\", reason=\"NodeAffinity\", service=\"prometheus-kube-state-metrics\", uid=\"1f4cb14c-70a0-45e7-9cd4-e57b3292287d\"}" timestamp=1689546041568
Jul 17 03:21:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:52.309Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_frequency_min_hertz\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546046709
Jul 17 03:21:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:52.462Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_packets_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth561c9aba\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546049337
Jul 17 03:21:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:52.525Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_queue_length\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth41c1253b\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546049337
Jul 17 03:21:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:52.540Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db82\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546039367
Jul 17 03:21:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:53.107Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_errors_total\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", err=\"ERR\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546050108
Jul 17 03:21:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:53.228Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_id\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"cnibr0\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546048328
Jul 17 03:21:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:53.264Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2dpath-pci\\\\x2d0000:81:00.0\\\\x2dscsi\\\\x2d0:2:0:0\\\\x2dpart5.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689546051179
Jul 17 03:21:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:53.267Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_commands_total\", asset=\"msh-tsgx08\", asset_id=\"65\", cmd=\"del\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546050108
Jul 17 03:21:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:53.286Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-netns-cni\\\\x2d253b3116\\\\x2d9292\\\\x2d2bfc\\\\x2df01e\\\\x2db683af436678.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689546051179
Jul 17 03:21:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:53.366Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_persistentvolume_created\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", persistentvolume=\"pvc-4aa77b43-ba02-43ea-8bdf-c9edeffb4444\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546048663
Jul 17 03:21:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:53.380Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx12\", asset_id=\"113\", cpu=\"28\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546043391
Jul 17 03:21:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:53.425Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_packets_total\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens7f0np0\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546043391
Jul 17 03:21:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:53.432Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_down_changes_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth50cee5b3\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546049337
Jul 17 03:21:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:53.455Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"vmtoolsd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"service\"}" timestamp=1689546051179
Jul 17 03:21:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:53.463Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Udp_Bit\", app_name=\"sapp4\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", vsysId=\"1\"}" timestamp=1689546040470
Jul 17 03:21:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:53.488Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_replicas\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"ipmi-exporter-6f8464dd94\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546048663
Jul 17 03:21:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:53.548Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx12\", asset_id=\"113\", cpu=\"20\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546043391
Jul 17 03:21:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:53.553Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-93005333\\\\x2d2bbb\\\\x2d4ce3\\\\x2da6b6\\\\x2da5b632269832-volume\\\\x2dsubpaths-telegraf-telegraf\\\\x2dsecurity-0.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689546045496
Jul 17 03:21:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:53.614Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_packets_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"eno2\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546039409
Jul 17 03:21:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:53.632Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2dpartuuid-93356bef\\\\x2d0028\\\\x2d4f3a\\\\x2da71f\\\\x2d9a3b50706d03.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689546040884
Jul 17 03:21:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:53.642Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sshd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"service\"}" timestamp=1689546040884
Jul 17 03:21:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:53.723Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db245\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546039367
Jul 17 03:21:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:53.730Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-binfmt.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"oneshot\", type=\"service\"}" timestamp=1689546045496
Jul 17 03:21:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:53.762Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_ulimits_soft\", asset=\"msh-tsgx08\", asset_id=\"65\", container=\"telegraf\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", ulimit=\"max_open_files\"}" timestamp=1689546034795
Jul 17 03:21:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:53.802Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_created\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-headless\", uid=\"869463cd-4818-41dc-81a2-fd7561926bcb\"}" timestamp=1689546044478
Jul 17 03:21:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:53.836Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_terminated\", asset=\"pcap-tsgx11\", asset_id=\"115\", container=\"helm\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-mrzcpd-exporter-57lwk\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"af61fc36-7433-4c28-9879-b0c3e4aa66b1\"}" timestamp=1689546044478
Jul 17 03:21:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:53.867Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_terminated_reason\", asset=\"pcap-tsgx11\", asset_id=\"115\", container=\"helm\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-cm-policy-local-cache-xlkqt\", project=\"TSG-OS\", project_id=\"7\", reason=\"Completed\", service=\"prometheus-kube-state-metrics\", uid=\"4f506bcb-6f14-4c9c-9398-6b207614efb5\"}" timestamp=1689546044478
Jul 17 03:21:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:53.976Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_transmit_wait_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"mlx5_6\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546039409
Jul 17 03:21:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:54.002Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"overlay\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-2fa54e99b592a2324df2f3fe0fa3f21476666e6b107d47b83794bffeb70d8c9f-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689546040884
Jul 17 03:21:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:54.066Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", cpu=\"77\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546049337
Jul 17 03:21:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:54.100Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx08\", asset_id=\"65\", cpu=\"84\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546048328
Jul 17 03:21:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:54.119Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx08\", asset_id=\"65\", cpu=\"73\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546048328
Jul 17 03:21:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:54.251Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_cache\", asset=\"pcap-tsgx07\", asset_id=\"85\", container=\"lb-tcp-80\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546043471
Jul 17 03:21:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:54.274Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cpu=\"51\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546039409
Jul 17 03:21:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:54.279Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"pcap-tsgx07\", asset_id=\"85\", condition=\"Progressing\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"tsg-traffic-engine-numa-0-firewall\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"true\"}" timestamp=1689546042058
Jul 17 03:21:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:54.300Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_procs_running\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546039409
Jul 17 03:21:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:54.343Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled_time\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"ipmi-exporter-6f8464dd94-ghpck\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"ff360f8e-d8b6-4451-9e19-c901d1e7ae8c\"}" timestamp=1689546048950
Jul 17 03:21:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:54.363Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx08\", asset_id=\"65\", cpu=\"35\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546048328
Jul 17 03:21:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:54.410Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"traefik-64f55bb67d-bcw8n\", project=\"TSG-OS\", project_id=\"7\", reason=\"Evicted\", service=\"prometheus-kube-state-metrics\", uid=\"b9686156-a602-47ce-b798-d231f10c8f92\"}" timestamp=1689546042058
Jul 17 03:21:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:54.594Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"36\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546039409
Jul 17 03:21:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:54.620Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_commands_rejected_calls_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cmd=\"unlink\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546045184
Jul 17 03:21:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:54.987Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_protocol_type\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth211eccff\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546039409
Jul 17 03:21:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:55.118Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_oom_events_total\", asset=\"msh-tsgx12\", asset_id=\"113\", container=\"kube-multus\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546029300
Jul 17 03:21:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:55.165Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_persistentvolume_status_phase\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", persistentvolume=\"pvc-4aa77b43-ba02-43ea-8bdf-c9edeffb4444\", phase=\"Available\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546048663
Jul 17 03:21:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:55.210Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_carrier_total\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens1f1np1\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546043391
Jul 17 03:21:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:55.231Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_local_link_integrity_errors_total\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_9\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546048328
Jul 17 03:21:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:55.409Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_threads\", asset=\"msh-tsgx12\", asset_id=\"113\", container=\"kubernetes-dashboard\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546030132
Jul 17 03:21:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:55.432Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"prometheus-kube-state-metrics\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546048663
Jul 17 03:21:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:55.451Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", container=\"firewall\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"cadvisor\", failure_type=\"pgmajfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"hierarchy\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689546045181
Jul 17 03:21:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:55.479Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cpu=\"47\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546039409
Jul 17 03:21:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:55.833Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_restart_policy\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-prometheus-fklsr\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", type=\"OnFailure\", uid=\"fdb8ee98-3736-4e4f-892f-7a3e7938e127\"}" timestamp=1689546042058
Jul 17 03:21:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:55.854Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"108\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546039409
Jul 17 03:21:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:55.895Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_memory_KReclaimable_bytes\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546039409
Jul 17 03:21:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:55.950Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_net_dev_group\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth55feb6e8\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546039409
Jul 17 03:21:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:55.977Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-b9686156\\\\x2da602\\\\x2d47ce\\\\x2db798\\\\x2dd231f10c8f92-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2dnn4bg.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689546040884
Jul 17 03:21:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:55.988Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cpu=\"92\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546039409
Jul 17 03:21:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:55.997Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"multi-user.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"target\"}" timestamp=1689546040884
Jul 17 03:21:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:56.011Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_size_bytes\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"shm\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/8b574de184cc170685ed96554d0520635bb250dd515d665688440f2cda822faf/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546039409
Jul 17 03:21:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:56.027Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_up\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens6f1np1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546039409
Jul 17 03:21:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:56.032Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_multicast_packets_received_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"mlx5_1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546039409
Jul 17 03:21:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:56.124Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_processes\", asset=\"pcap-tsgx07\", asset_id=\"85\", container=\"configmap-reloader\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546044863
Jul 17 03:21:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:56.148Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_address_assign_type\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"br_dign_c\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546039409
Jul 17 03:21:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:56.244Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"52\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546045883
Jul 17 03:21:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:56.251Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_mtu_bytes\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"br_dign_s\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546045883
Jul 17 03:21:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:56.272Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"74\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546045883
Jul 17 03:21:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:56.363Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_net_dev_group\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"eno33np0\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546045883
Jul 17 03:21:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:56.546Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_drop_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth2b692087\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546045883
Jul 17 03:21:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:56.727Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:e0-0000:e0:03.1-0000:e2:00.0-net-ens8f0np0.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689546043372
Jul 17 03:21:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:57.039Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"clickhouse_os_guest_nice_time_cpu11\", asset=\"msh-dellr740-olap001\", asset_id=\"11\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Clickhouse-msh-dellr740-olap001\", endpoint_id=\"71\", module=\"NC-Clickhouse\", module_id=\"29\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546055797
Jul 17 03:21:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:57.646Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_usage_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", container=\"prometheus-alertmanager\", cpu=\"total\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546045484
Jul 17 03:21:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:57.747Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"42\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546045883
Jul 17 03:21:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:59.806Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_errors_received_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_6\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546049083
Jul 17 03:21:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:59.861Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_spec_replicas\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-1-firewall-7f9c998578\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546047425
Jul 17 03:21:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:59.883Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"89\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546049083
Jul 17 03:21:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:59.955Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"8\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546049083
Jul 17 03:21:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:21:59.994Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"lm_sensors.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"oneshot\", type=\"service\"}" timestamp=1689546044879
Jul 17 03:22:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:00.153Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_rate_bytes_per_second\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_11\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546049083
Jul 17 03:22:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:00.231Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-mrzcpd-huge_pages.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689546044879
Jul 17 03:22:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:00.258Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"tmpfs\", name=\"var-lib-kubelet-pods-e38aa3df\\\\x2dc4d6\\\\x2d4584\\\\x2dbadd\\\\x2d68c48f35fb6d-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2dmx72q.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689546044879
Jul 17 03:22:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:00.497Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-adf38312f586f9789395221f46cd8b7f6a29e1378158d03549f9e4d136a25c13-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689546044879
Jul 17 03:22:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:00.531Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:20-0000:20:03.1-0000:21:00.1-net-ens2f1np1.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689546044879
Jul 17 03:22:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:00.731Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx01\", asset_id=\"51\", container=\"exporter\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Read\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546042899
Jul 17 03:22:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:00.807Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_packets_dropped_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"cadvisor\", interface=\"virtio_dign_c\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546052943
Jul 17 03:22:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:00.845Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_restart_policy\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"metrics-server-54dc485875-fzfcm\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", type=\"Always\", uid=\"b1c94976-83e8-41e4-85f6-2606e4291f65\"}" timestamp=1689546047425
Jul 17 03:22:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:00.853Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"17\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546049083
Jul 17 03:22:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:00.871Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"49\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546049083
Jul 17 03:22:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:00.886Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-b3efc35a\\\\x2d4844\\\\x2d47d9\\\\x2d82c9\\\\x2d7d335d0d4eaa-volume\\\\x2dsubpaths-config\\\\x2dvolume-telegraf-2.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689546044879
Jul 17 03:22:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:01.178Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2did-scsi\\\\x2d362cea7f085d4ec002c2b67bee386ff9d\\\\x2dpart1.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689546044879
Jul 17 03:22:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:01.368Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"msh-tsgx01\", asset_id=\"51\", container=\"dign-server-dns\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"stopped\"}" timestamp=1689546046307
Jul 17 03:22:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:01.547Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"msh-tsgx01\", asset_id=\"51\", condition=\"false\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-alertmanager-6df7f457b6-pffs9\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"e38aa3df-c4d6-4584-badd-68c48f35fb6d\"}" timestamp=1689546047425
Jul 17 03:22:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:01.559Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dracut-pre-mount.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"service\"}" timestamp=1689546044879
Jul 17 03:22:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:01.754Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"78\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546049083
Jul 17 03:22:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:01.792Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"81\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546049083
Jul 17 03:22:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:02.551Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"36\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546058318
Jul 17 03:22:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:02.590Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_carrier_total\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens1f1np1\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546058318
Jul 17 03:22:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:02.610Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-burstable.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"slice\"}" timestamp=1689546048447
Jul 17 03:22:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:02.760Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_nf_conntrack_stat_drop\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546058318
Jul 17 03:22:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:02.833Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_max_fds\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"rsyslog.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689546048447
Jul 17 03:22:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:02.869Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-88292df9a400697453f6b71cacb959f738a468b91fcfa47d9a0f641723a4f338.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"scope\"}" timestamp=1689546048447
Jul 17 03:22:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:02.898Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx02\", asset_id=\"63\", container=\"telegraf\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Total\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689546049199
Jul 17 03:22:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:02.973Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_errors_total\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", interface=\"ens7f1np1\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546051599
Jul 17 03:22:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:02.997Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx02\", asset_id=\"63\", cpu=\"29\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546058318
Jul 17 03:22:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:03.054Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_bytes_total\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens3f0np0\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546058318
Jul 17 03:22:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:03.131Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_waiting\", asset=\"msh-tsgx02\", asset_id=\"63\", container=\"exporter-merger\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"mrzcpd-exporter-5cf7f967cc-9zrnz\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"0b1be0e3-e71c-4f3b-83a7-1d29910105b8\"}" timestamp=1689546051412
Jul 17 03:22:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:03.510Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_reads_bytes_total\", asset=\"msh-tsgx02\", asset_id=\"63\", container=\"dign-server-web\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546051667
Jul 17 03:22:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:03.794Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-3dc9560d34854d0ab10b492557f9080698dc968713dde4efdee2f2f982e585a6-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689546048447
Jul 17 03:22:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:03.813Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db54\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546049269
Jul 17 03:22:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:03.827Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-virtual-net-veth2ff062e3.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689546048447
Jul 17 03:22:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:03.855Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"tmpfs\", name=\"var-lib-kubelet-pods-03938281\\\\x2de348\\\\x2d4392\\\\x2da795\\\\x2df2d53f05d546-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2d5qkm7.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689546048447
Jul 17 03:22:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:03.870Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-7336ebc13f6a18441b091a169d603ade439ceb80fc0ae83a8ac2dc7d3ffd4569-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689546048447
Jul 17 03:22:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:03.872Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_avail_bytes\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"shm\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/f3e32b6b99348bfd481a4b517930a34254f7d5dd8ed8a945bc14711be6aae797/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546058318
Jul 17 03:22:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:03.991Z caller=dedupe.go:112 component=remote level=info remote_name=52db6f url=http://10.10.10.159:10090/nz-agent/prometheus/proxy/api/v1/write msg="Dropped sample for series that was not explicitly dropped via relabelling" ref=4436650
Jul 17 03:22:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:04.189Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"prometheus_target_scrape_pool_exceeded_label_limits_total\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", instance=\"10.10.20.159:10090\", type=\"global\"}" timestamp=1689546062990
Jul 17 03:22:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:04.191Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_frequency_max_hertz\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", cpu=\"13\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-dt001\", endpoint_id=\"225\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546062987
Jul 17 03:22:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:04.261Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_drop_total\", asset=\"msh-dellr740-olap005\", asset_id=\"19\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"em4\", endpoint=\"NC-Spark-msh-dellr740-olap005\", endpoint_id=\"17\", module=\"NC-Spark\", module_id=\"47\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546063391
Jul 17 03:22:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:04.449Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"prometheus_http_request_duration_seconds_bucket\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", handler=\"/api/v1/label/:name/values\", instance=\"10.10.20.159:10090\", le=\"0.1\", type=\"global\"}" timestamp=1689546062990
Jul 17 03:22:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:04.482Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-dt001\", endpoint_id=\"225\", module=\"OLAP-Node-Exporter\", module_id=\"51\", name=\"23\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\", type=\"Processor\"}" timestamp=1689546062987
Jul 17 03:22:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:04.512Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sdb1\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-dt001\", endpoint_id=\"225\", fstype=\"xfs\", module=\"OLAP-Node-Exporter\", module_id=\"51\", mountpoint=\"/home\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546062987
Jul 17 03:22:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:06.638Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_memory_SReclaimable_bytes\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546063910
Jul 17 03:22:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:06.743Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"twa-tsgx01\", asset_id=\"91\", condition=\"Progressing\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"tsg-traffic-engine-numa-1-firewall\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"true\"}" timestamp=1689546061227
Jul 17 03:22:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:06.871Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_drop_total\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens4f2np2\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546063910
Jul 17 03:22:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:06.883Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_rss\", asset=\"twa-tsgx01\", asset_id=\"91\", container=\"dign-server-web\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546061300
Jul 17 03:22:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:06.906Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"local-fs.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"target\"}" timestamp=1689546053582
Jul 17 03:22:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:07.251Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"twa-tsgx01\", asset_id=\"91\", condition=\"true\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", pod=\"dashboard-metrics-scraper-5b94897dc9-xrbx4\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"d73900af-89f6-4478-880d-06274d9e40b5\"}" timestamp=1689546061227
Jul 17 03:22:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:07.305Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_name_assign_type\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens3f1np1\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546063910
Jul 17 03:22:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:07.324Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cpu=\"111\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546063910
Jul 17 03:22:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:07.663Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_alloc_bytes_total\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546063910
Jul 17 03:22:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:07.694Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_observed_generation\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"cm-policy-local-cache-redis-master\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546061227
Jul 17 03:22:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:07.717Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_netstat_Icmp6_InMsgs\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546063910
Jul 17 03:22:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:07.723Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_net_dev_group\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"bond0\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546063910
Jul 17 03:22:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:07.750Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cpu=\"6\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546063910
Jul 17 03:22:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:07.798Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-modules-load.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"oneshot\", type=\"service\"}" timestamp=1689546053582
Jul 17 03:22:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:07.922Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_labels\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", job_name=\"helm-install-tsg-traffic-engine-numa-0\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546061227
Jul 17 03:22:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:07.948Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cryptsetup-pre.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"target\"}" timestamp=1689546053582
Jul 17 03:22:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:07.971Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-virtual-net-vetha2b94414.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689546053582
Jul 17 03:22:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:08.013Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_bytes_total\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens6f0np0\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546063910
Jul 17 03:22:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:08.275Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db187\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546058290
Jul 17 03:22:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:08.352Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-2eabba01240a8c268f3e154d066d3488c73f85fff9d466bdf6c3b04cd25b5f30-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689546053582
Jul 17 03:22:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:08.383Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kdump.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"oneshot\", type=\"service\"}" timestamp=1689546053582
Jul 17 03:22:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:08.407Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_NameNode_UnderReplicatedBlocks\", asset=\"twa-dellr740-dt002\", asset_id=\"39\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-Hadoop-twa-dellr740-dt002-NameNode\", endpoint_id=\"337\", module=\"DC-Hadoop\", module_id=\"53\", name=\"FSNamesystemState\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546067246
Jul 17 03:22:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:08.428Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"cortex_ingester_tsdb_compactions_failed_total\", agent_name=\"msh-kvm-nz001\", application=\"cortex\", datacenter=\"MSH\", instance=\"127.0.0.1:10093\"}" timestamp=1689546067267
Jul 17 03:22:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:08.459Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"prometheus_sd_file_scan_duration_seconds\", agent_name=\"msh-kvm-nz001\", application=\"cortex\", datacenter=\"MSH\", instance=\"127.0.0.1:10093\", quantile=\"0.5\"}" timestamp=1689546067267
Jul 17 03:22:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:08.532Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-remount-fs.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689546053582
Jul 17 03:22:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:08.612Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx01\", asset_id=\"91\", container=\"dashboard-metrics-scraper\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Discard\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546053941
Jul 17 03:22:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:08.663Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_rss\", asset=\"twa-tsgx01\", asset_id=\"91\", container=\"firewall\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689546045912
Jul 17 03:22:09 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:09.357Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"scrape_samples_post_metric_relabeling\", asset=\"msh-dellr740-olap002\", asset_id=\"13\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"Galaxy-job-admin-msh-dellr740-olap002\", endpoint_id=\"49\", instance=\"192.168.20.222:8184\", job=\"49\", module=\"Galaxy-job-admin\", module_id=\"17\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546067039
Jul 17 03:22:09 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:09.821Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Append_95th_percentile\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Server\"}" timestamp=1689546068804
Jul 17 03:22:09 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:09.824Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_compactionsFailedCount\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"assessment_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"c16a0ad8b16e365f0379acba25fc24b7\", table=\"index_partfile_assessment_hos_bucket\", type=\"index\"}" timestamp=1689546068804
Jul 17 03:22:09 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:09.839Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_writeRequestCount\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"knowledge_base_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"ebc2debc1fb19c5df58bdfbbd410b63e\", table=\"index_time_knowledge_base_hos_bucket\", type=\"index\"}" timestamp=1689546068804
Jul 17 03:22:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:10.029Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"scrape_samples_scraped\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"Galaxy-hos-nginx-pcap-dellr740-dt001\", endpoint_id=\"205\", instance=\"192.168.10.193:9914\", job=\"205\", module=\"Galaxy-hos-nginx\", module_id=\"13\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546069632
Jul 17 03:22:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:10.050Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_numBytesCompactedCount\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"default\", project=\"OLAP\", project_id=\"3\", region=\"b80f77b3ae41c2c5dc46743c217a463a\", table=\"index_filename_default\", type=\"index\"}" timestamp=1689546068804
Jul 17 03:22:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:10.070Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_numBytesCompactedCount\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"a3eb769a8c2d325e0a97838f6786a569\", table=\"session_record_hos_bucket\", type=\"data\"}" timestamp=1689546068804
Jul 17 03:22:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:10.198Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_appendCount\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"troubleshooting_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"bf2a458b5ee504849c8f27ed46a87a35\", table=\"index_filename_troubleshooting_hos_bucket\", type=\"index\"}" timestamp=1689546068804
Jul 17 03:22:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:10.260Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_appendCount\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"firewall_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"25eeeb642a8f4f09df9d4f773c55d04f\", table=\"index_filename_firewall_hos_bucket\", type=\"index\"}" timestamp=1689546068804
Jul 17 03:22:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:10.332Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_compactionsFailedCount\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"troubleshooting_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"85895fbc8b61b2aea3a479507705491f\", table=\"index_filename_troubleshooting_hos_bucket\", type=\"index\"}" timestamp=1689546068804
Jul 17 03:22:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:10.365Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_incrementCount\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"53a0c4e4a0d9c36785db16c7f328260b\", table=\"troubleshooting_hos_bucket\", type=\"data\"}" timestamp=1689546068804
Jul 17 03:22:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:10.604Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_minStoreFileAge\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"a613a96655d52a6bff90b8513556bb77\", table=\"knowledge_base_hos_bucket\", type=\"data\"}" timestamp=1689546068804
Jul 17 03:22:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:10.622Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_numFilesCompactedCount\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"troubleshooting_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"ab308cf77633e8031646d30c5c5926af\", table=\"index_time_troubleshooting_hos_bucket\", type=\"index\"}" timestamp=1689546068804
Jul 17 03:22:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:10.829Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_numFilesCompactedCount\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"troubleshooting_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"d2114d26e64525f0229a1f51892d452d\", table=\"index_partfile_troubleshooting_hos_bucket\", type=\"index\"}" timestamp=1689546068804
Jul 17 03:22:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:11.420Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_compactionsCompletedCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"knowledge_base_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"04ab4d5780512f01a21447ac053d6154\", table=\"index_filename_knowledge_base_hos_bucket\", type=\"index\"}" timestamp=1689546070148
Jul 17 03:22:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:11.797Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_GcCountConcurrentMarkSweep\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"JvmMetrics\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546070148
Jul 17 03:22:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:11.890Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Append_TimeRangeCount_10_30\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Server\"}" timestamp=1689546070148
Jul 17 03:22:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:12.222Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_filteredReadRequestCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"knowledge_base_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"f3f09140dd361cd54664033e5fd9a301\", table=\"index_partfile_knowledge_base_hos_bucket\", type=\"index\"}" timestamp=1689546070148
Jul 17 03:22:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:12.281Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_numReferenceFiles\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"firewall_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"56d566e1b992b71c05beb00e33d04688\", table=\"index_time_firewall_hos_bucket\", type=\"index\"}" timestamp=1689546070148
Jul 17 03:22:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:12.288Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_numReferenceFiles\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"session_record_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"fe5edc011eaddb1eb1c7961140d3dbc5\", table=\"index_filename_session_record_hos_bucket\", type=\"index\"}" timestamp=1689546070148
Jul 17 03:22:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:12.338Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_193_1_metric_majorCompactionOutputFileCount_95th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546070148
Jul 17 03:22:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:12.353Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_3_metric_deleteBatchTime_min\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546070148
Jul 17 03:22:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:12.477Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_session_record_hos_bucket_metric_flushOutputSize_90th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546070148
Jul 17 03:22:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:12.491Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_default_metric_tableWriteQueryPerSecond_count\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546070148
Jul 17 03:22:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:12.702Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_maxFlushQueueSize\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"default\", project=\"OLAP\", project_id=\"3\", region=\"18f73459ccacec184acdb88b607bc5eb\", table=\"index_time_default\", type=\"index\"}" timestamp=1689546070148
Jul 17 03:22:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:12.772Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_heap_released_bytes\", asset=\"twa-dellr740-dt002\", asset_id=\"39\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"NC-Gohangout-twa-dellr740-dt002\", endpoint_id=\"179\", module=\"DC-Gohangout\", module_id=\"35\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546071525
Jul 17 03:22:13 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:13.106Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_1_metric_appendTime_median\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546070148
Jul 17 03:22:13 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:13.126Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_maxFlushQueueSize\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"session_record_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"27e64d59c20926b648fd0e4254c5cf70\", table=\"index_partfile_session_record_hos_bucket\", type=\"index\"}" timestamp=1689546070148
Jul 17 03:22:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:16.127Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_info\", asset=\"twa-tsgx07\", asset_id=\"103\", configmap=\"extension-apiserver-authentication\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546068076
Jul 17 03:22:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:16.421Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_annotations\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-node-exporter-pfqsj\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"85ce1d65-33c2-41b7-81ab-b25e87a182c7\"}" timestamp=1689546068076
Jul 17 03:22:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:16.441Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"110\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546061230
Jul 17 03:22:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:16.465Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"106\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546061230
Jul 17 03:22:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:16.480Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"78\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546061230
Jul 17 03:22:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:16.544Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_ready_replicas\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"prometheus-alertmanager-6df7f457b6\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546068076
Jul 17 03:22:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:16.559Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_labels\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-headless\", uid=\"5b8df33d-b6e9-4ca2-af20-9b144f8b9f96\"}" timestamp=1689546068076
Jul 17 03:22:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:16.569Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"phy_tx_error_total\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", dev=\"ens7f0np0\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"mrzcpd-exporter\"}" timestamp=1689546066692
Jul 17 03:22:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:16.588Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db212\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546066027
Jul 17 03:22:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:16.594Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_constraint_errors_received_total\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_8\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546061230
Jul 17 03:22:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:16.631Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2dpartuuid-e9354034\\\\x2d2447\\\\x2d4283\\\\x2dbdec\\\\x2d7404ab8488d4.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689546067046
Jul 17 03:22:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:16.708Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"17\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546061230
Jul 17 03:22:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:19.634Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_ready_replicas\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"systemd-exporter-7f549d9485\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546073949
Jul 17 03:22:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:19.652Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"106\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546077612
Jul 17 03:22:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:19.870Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"20\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546077612
Jul 17 03:22:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:19.897Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"twa-tsgx06\", asset_id=\"101\", condition=\"unknown\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-prometheus-l7d72\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"147bad19-ad92-40e0-9b33-cd00383880da\"}" timestamp=1689546073949
Jul 17 03:22:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:19.916Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"2\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546077612
Jul 17 03:22:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:20.104Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"67\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546077612
Jul 17 03:22:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:20.122Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"47\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546077612
Jul 17 03:22:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:20.135Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_device_id\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"br_dign_c\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546077612
Jul 17 03:22:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:20.179Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_scrape_collector_success\", asset=\"twa-tsgx06\", asset_id=\"101\", collector=\"zfs\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546077612
Jul 17 03:22:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:20.284Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_ready_replicas\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"prometheus-alertmanager-6df7f457b6\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546073828
Jul 17 03:22:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:20.301Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"local-fs-pre.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"target\"}" timestamp=1689546064356
Jul 17 03:22:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:20.314Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_annotations\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.tsg-traffic-engine-numa-0.v2\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546073828
Jul 17 03:22:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:20.547Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"125\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546068263
Jul 17 03:22:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:21.143Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"48\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546077612
Jul 17 03:22:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:21.169Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"0\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546077612
Jul 17 03:22:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:21.236Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"88\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546077612
Jul 17 03:22:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:21.331Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_annotations\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", job_name=\"helm-install-configmap-reloader\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546073828
Jul 17 03:22:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:21.349Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_status_failed\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", job_name=\"helm-install-traefik\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546073828
Jul 17 03:22:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:21.427Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_threads\", asset=\"pcap-tsgx10\", asset_id=\"89\", container=\"firewall\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689546068857
Jul 17 03:22:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:21.658Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"126\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546068263
Jul 17 03:22:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:21.681Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"k3s.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689546070657
Jul 17 03:22:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:21.748Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-38e3a560b644b18ec47fddbe282b76d0f18f92096cb0d76cc9e55d5a2b9dad2b.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"scope\"}" timestamp=1689546070657
Jul 17 03:22:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:21.775Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:e0-0000:e0:03.1-0000:e2:00.0-infiniband_mad-umad10.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689546070657
Jul 17 03:22:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:21.786Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_labels\", asset=\"pcap-tsgx10\", asset_id=\"89\", configmap=\"prometheus-server\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546073828
Jul 17 03:22:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:21.800Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-ens4f3np3.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689546070657
Jul 17 03:22:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:21.976Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_threads\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"OLAP-Node-Exporter-twa-dellr740-dt001\", endpoint_id=\"257\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546081405
Jul 17 03:22:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:22.015Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_commands_rejected_calls_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cmd=\"slowlog\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546074840
Jul 17 03:22:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:22.306Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db166\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546074840
Jul 17 03:22:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:22.464Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_writes_merged_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda3\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546071200
Jul 17 03:22:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:23.458Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"druid_index_task_healthy_flag\", asset=\"msh-dellr740-olap004\", asset_id=\"17\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Spark-msh-dellr740-olap004\", endpoint_id=\"15\", module=\"NC-Spark\", module_id=\"47\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546082254
Jul 17 03:22:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:23.556Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"msh-tsgx09\", asset_id=\"55\", condition=\"true\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-kube-state-metrics-6b768c7bf8-db9cd\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"1c96629c-a0c6-4e4c-8e97-b6bbe91aa6d9\"}" timestamp=1689546080068
Jul 17 03:22:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:23.577Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sysstat.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689546078416
Jul 17 03:22:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:23.587Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_metadata_generation\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"configmap-reloader-6c55bbc49f\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546080068
Jul 17 03:22:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:24.053Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx09\", asset_id=\"55\", cpu=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546074790
Jul 17 03:22:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:24.098Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_dormant\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens6f1np1\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546074790
Jul 17 03:22:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:24.123Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_state_id\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_4\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546074790
Jul 17 03:22:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:24.131Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"msh-tsgx09\", asset_id=\"55\", cpu=\"31\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546074790
Jul 17 03:22:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:24.254Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_cpu_shares\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"prometheus-server-configmap-reload\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546083419
Jul 17 03:22:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:24.278Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db243\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546077019
Jul 17 03:22:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:24.601Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_inodes_free\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/run\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546073835
Jul 17 03:22:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:24.658Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-9e233e9f\\\\x2dde40\\\\x2d4ebd\\\\x2dabfb\\\\x2de6360a2d9bb6-volumes-kubernetes.io\\\\x7esecret-config.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689546078416
Jul 17 03:22:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:24.667Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_spec_paused\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"mrzcpd-exporter\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546080068
Jul 17 03:22:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:24.765Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_metadata_resource_version\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"kubernetes-dashboard-certs\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546080068
Jul 17 03:22:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:24.773Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db1\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546077019
Jul 17 03:22:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:24.850Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"dign-server-dns\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Sync\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546070986
Jul 17 03:22:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:24.857Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"pcap-tsgx12\", asset_id=\"117\", condition=\"false\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", pod=\"kubernetes-dashboard-66dd5b456-rs7kq\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"cbd312de-6144-4424-8813-fba2b3b16e9b\"}" timestamp=1689546071122
Jul 17 03:22:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:24.858Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_start_time\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"ipmi-exporter-6f8464dd94-n7w69\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"9e233e9f-de40-4ebd-abfb-e6360a2d9bb6\"}" timestamp=1689546080068
Jul 17 03:22:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:24.881Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_created\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-0-firewall-6cb6b98c7c\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546080068
Jul 17 03:22:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:24.885Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_created\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"prometheus-kube-state-metrics-6b768c7bf8\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546080068
Jul 17 03:22:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:24.904Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_storageclass_created\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", storageclass=\"local-path\"}" timestamp=1689546071122
Jul 17 03:22:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:24.953Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_up_changes_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"eno35np2\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546080160
Jul 17 03:22:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:25.042Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2duuid-39cf7e4e\\\\x2d9434\\\\x2d42ca\\\\x2dbe9f\\\\x2d6ca495c8abdc.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689546071102
Jul 17 03:22:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:25.097Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-af33c36f9afb63daa7065372a5ea8315b79f89e1869cf7221a45c81aa5a2a41c-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689546078416
Jul 17 03:22:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:25.110Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"8\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546080160
Jul 17 03:22:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:25.246Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_replicas\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"coredns-7cbc7ddf87\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546080068
Jul 17 03:22:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:25.365Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"phy_rx_error_total\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", dev=\"virtio_dign_s\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"mrzcpd-exporter\"}" timestamp=1689546080889
Jul 17 03:22:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:25.383Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx09\", asset_id=\"55\", cpu=\"54\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546074790
Jul 17 03:22:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:25.426Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_memory_HardwareCorrupted_bytes\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546074790
Jul 17 03:22:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:25.483Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Udp_Link_S2C_ALL\", app_name=\"sapp4\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", vsysId=\"1\"}" timestamp=1689546072900
Jul 17 03:22:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:25.677Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_reservation_limit_bytes\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"prometheus-node-exporter\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546083419
Jul 17 03:22:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:25.987Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_queue_length\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth1e7f8cb3\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546075810
Jul 17 03:22:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:26.024Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_threads\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"prometheus-alertmanager\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546069008
Jul 17 03:22:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:26.041Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-pushgateway-bd476bcf9-z79gb\", project=\"TSG-OS\", project_id=\"7\", reason=\"Evicted\", service=\"prometheus-kube-state-metrics\", uid=\"8ea47e8a-bb11-4592-aee9-0458e6cbc258\"}" timestamp=1689546071122
Jul 17 03:22:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:26.049Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_restart_policy\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", pod=\"kubernetes-dashboard-66dd5b456-bgg44\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", type=\"Always\", uid=\"c8850d1a-9bcd-4bf0-adad-91f75a60d9cc\"}" timestamp=1689546078744
Jul 17 03:22:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:26.069Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_compressed_total\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"eno33np0\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546075810
Jul 17 03:22:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:26.085Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_scrape_collector_success\", asset=\"twa-tsgx09\", asset_id=\"107\", collector=\"netclass\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546075810
Jul 17 03:22:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:26.089Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_info\", address=\"b8:ce:f6:88:53:e3\", asset=\"twa-tsgx09\", asset_id=\"107\", broadcast=\"ff:ff:ff:ff:ff:ff\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens8f1np1\", duplex=\"unknown\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operstate=\"down\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546075810
Jul 17 03:22:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:26.136Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-692f1a13\\\\x2d48e0\\\\x2d47b7\\\\x2dad46\\\\x2da5b348bd402b-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2dgbthj.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689546071102
Jul 17 03:22:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:26.210Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_info\", address=\"b0:7b:25:d4:98:a6\", asset=\"pcap-tsgx12\", asset_id=\"117\", broadcast=\"ff:ff:ff:ff:ff:ff\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"eno1\", duplex=\"full\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operstate=\"up\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546080160
Jul 17 03:22:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:26.224Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dnf-makecache.timer\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"timer\"}" timestamp=1689546071102
Jul 17 03:22:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:26.364Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_mspan_inuse_bytes\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546075810
Jul 17 03:22:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:26.470Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_down_changes_total\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth4e29dafa\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546075810
Jul 17 03:22:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:26.513Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"emergency.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"service\"}" timestamp=1689546073907
Jul 17 03:22:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:26.856Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_labels\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_endpoint=\"mrzcpd-exporter\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546078744
Jul 17 03:22:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:26.872Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_type\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.tsg-traffic-engine-numa-1.v1\", service=\"prometheus-kube-state-metrics\", type=\"helm.sh/release.v1\"}" timestamp=1689546078744
Jul 17 03:22:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:26.887Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546075810
Jul 17 03:22:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:26.901Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens1f1np1\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546075810
Jul 17 03:22:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:26.948Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_io_current\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"overlay_0-47\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546077795
Jul 17 03:22:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:26.984Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_dormant\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth16c89716\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546075810
Jul 17 03:22:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:26.995Z caller=dedupe.go:112 component=remote level=error remote_name=52db6f url=http://10.10.10.159:10090/nz-agent/prometheus/proxy/api/v1/write msg="non-recoverable error" count=1000 exemplarCount=0 err="server returned HTTP status 400 : out of order sample"
Jul 17 03:22:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:27.050Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_files_free\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"shm\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/281d3d5aca4b2d4dc3be2e9d6723bee656017aa58ce0b5b6792429535f280336/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546075810
Jul 17 03:22:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:27.232Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas_updated\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"ipmi-exporter\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546078744
Jul 17 03:22:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:27.452Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_cache\", asset=\"twa-tsgx09\", asset_id=\"107\", container=\"mrapm-device\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546079639
Jul 17 03:22:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:27.484Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_info\", asset=\"twa-tsgx09\", asset_id=\"107\", created_by_kind=\"Job\", created_by_name=\"helm-install-tsg-traffic-engine-numa-1\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", host_ip=\"10.10.30.169\", host_network=\"false\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", node=\"localhost\", nz_agent_id=\"3\", pod=\"helm-install-tsg-traffic-engine-numa-1-ngk8g\", pod_ip=\"10.42.0.116\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"72bb6b38-6d91-42a2-93c7-d542624b5d56\"}" timestamp=1689546078744
Jul 17 03:22:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:27.663Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx09\", asset_id=\"107\", container=\"dign-server-dns\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Discard\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546082132
Jul 17 03:22:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:27.855Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_reservation_limit_bytes\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"telegraf-security\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689546084841
Jul 17 03:22:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:27.866Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_status_active\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", job_name=\"helm-install-prometheus\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546078744
Jul 17 03:22:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:27.883Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_max_usage_bytes\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"firewall\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689546084794
Jul 17 03:22:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:27.903Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_vl15_dropped_total\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546075810
Jul 17 03:22:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:27.922Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", phase=\"Succeeded\", pod=\"tsg-traffic-engine-numa-0-firewall-556f86477-7wv6g\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"0b0292d1-7c6a-4fad-a1f9-f14e3c7cb4f8\"}" timestamp=1689546075682
Jul 17 03:22:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:28.137Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_fifo_total\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"bond0\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546075810
Jul 17 03:22:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:28.156Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"abrtd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"service\"}" timestamp=1689546073907
Jul 17 03:22:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:28.179Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2dlabel-TGS\\\\x2dOS\\\\x2dSYSROOT.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689546073907
Jul 17 03:22:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:28.252Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_completion_time\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-tsg-traffic-engine-numa-0-dt6k8\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"3456946b-cbfb-47c0-ab79-75794f863687\"}" timestamp=1689546075682
Jul 17 03:22:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:28.262Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"96\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546082882
Jul 17 03:22:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:28.396Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_created\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"local-path-provisioner-957fdf8bc-hswhg\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"99c87350-195b-4ca1-93bd-9a804bf4d300\"}" timestamp=1689546075682
Jul 17 03:22:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:28.409Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Tcp_Lost_Bit\", app_name=\"sapp4\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", vsysId=\"1\"}" timestamp=1689546084090
Jul 17 03:22:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:28.425Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"107\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546082882
Jul 17 03:22:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:28.466Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_observed_generation\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"prometheus-server-6ff784d758\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546075682
Jul 17 03:22:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:28.582Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"70\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546082882
Jul 17 03:22:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:28.652Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_dmi_info\", asset=\"msh-tsgx10\", asset_id=\"69\", bios_date=\"07/08/2020\", bios_release=\"1.4\", bios_version=\"1.4.78\", board_name=\"0590KW\", board_version=\"A01\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", product_sku=\"SKU=NotProvided;ModelName=\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546082882
Jul 17 03:22:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:28.681Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_size_bytes\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda5\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", fstype=\"ext4\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/home\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546082882
Jul 17 03:22:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:28.791Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_waiting\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"helm\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-prometheus-fbsht\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"bb5f85db-6a6f-4d8a-9d37-be7ad4071f86\"}" timestamp=1689546075682
Jul 17 03:22:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:28.813Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db173\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546079446
Jul 17 03:22:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:28.914Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"49\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546082882
Jul 17 03:22:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:28.944Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_queue_length\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens8f0np0\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546082882
Jul 17 03:22:29 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:29.048Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"cortex_kv_request_duration_seconds_bucket\", agent_name=\"msh-kvm-nz001\", application=\"loki\", datacenter=\"MSH\", instance=\"127.0.0.1:13100\", kv_name=\"ingester-lifecycler\", le=\"0.01\", operation=\"CAS\", role=\"primary\", status_code=\"200\", type=\"memberlist\"}" timestamp=1689546088327
Jul 17 03:22:29 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:29.217Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db26\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546079446
Jul 17 03:22:29 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:29.881Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"pcap-tsgx02\", asset_id=\"75\", cpu=\"34\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx02\", endpoint_id=\"469\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546076442
Jul 17 03:22:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:31.401Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_spec_parallelism\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", job_name=\"helm-install-mrzcpd-exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546084068
Jul 17 03:22:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:31.430Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_labels\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"k3s-serving\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546084068
Jul 17 03:22:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:31.523Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_info\", address=\"86:29:77:7b:47:03\", asset=\"twa-tsgx03\", asset_id=\"95\", broadcast=\"ff:ff:ff:ff:ff:ff\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"virtio_dign_s\", duplex=\"full\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operstate=\"unknown\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546079093
Jul 17 03:22:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:31.587Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"nis-domainname.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689546086907
Jul 17 03:22:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:31.632Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"jvm_buffer_pool_used_bytes\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-Yarn-twa-dellr740-dt001-NodeManager\", endpoint_id=\"363\", module=\"DC-Yarn\", module_id=\"1\", nz_agent_id=\"3\", pool=\"mapped\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546091354
Jul 17 03:22:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:31.651Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"71\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546079093
Jul 17 03:22:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:31.785Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"rescue.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"target\"}" timestamp=1689546082644
Jul 17 03:22:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:31.814Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_created\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"configmap-reloader\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546090436
Jul 17 03:22:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:31.842Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_threads_max\", asset=\"twa-tsgx03\", asset_id=\"95\", container=\"prometheus-pushgateway\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546066839
Jul 17 03:22:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:31.857Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_ready\", asset=\"twa-tsgx03\", asset_id=\"95\", container=\"telegraf-security\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", pod=\"tsg-traffic-engine-numa-1-firewall-dc9c7db68-v2sqf\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"1df6bd2b-0fea-4549-94c2-ddadc3bbc3e4\"}" timestamp=1689546090436
Jul 17 03:22:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:31.879Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"25\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546079093
Jul 17 03:22:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:31.896Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-1a454423\\\\x2da4c8\\\\x2d40de\\\\x2d896e\\\\x2d6655fed1aa68-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2drptjp.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689546086907
Jul 17 03:22:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:31.958Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_ulimits_soft\", asset=\"twa-tsgx03\", asset_id=\"95\", container=\"dign-client\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", ulimit=\"max_open_files\"}" timestamp=1689546072691
Jul 17 03:22:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:31.975Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_errors_received_total\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_6\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546079093
Jul 17 03:22:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:32.067Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_scrape_collector_duration_seconds\", asset=\"twa-tsgx03\", asset_id=\"95\", collector=\"infiniband\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546079093
Jul 17 03:22:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:32.084Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"99\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546080791
Jul 17 03:22:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:32.098Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"dashboard-metrics-scraper\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Async\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546071961
Jul 17 03:22:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:32.111Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_load_average_10s\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"dign-server-dns\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546076376
Jul 17 03:22:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:32.137Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"56\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546079093
Jul 17 03:22:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:32.173Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"44\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546079093
Jul 17 03:22:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:32.201Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_changes_total\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth5cd72b5c\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546079093
Jul 17 03:22:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:32.209Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db164\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546090141
Jul 17 03:22:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:32.226Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-a461d000f2d8b982cb9cb1e3d2902e1a2db16eae8ef0f9faaf4f810aa95c7b1d-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689546086907
Jul 17 03:22:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:32.234Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_rss\", asset=\"msh-tsgx05\", asset_id=\"67\", container=\"prometheus-alertmanager\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546063791
Jul 17 03:22:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:32.240Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_max_usage_bytes\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"prometheus-pushgateway\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546066634
Jul 17 03:22:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:32.241Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_avail_bytes\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"shm\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/b95df9c2544e432042325471d25f921b8246e5012d5337d6737d20aa6d480cf1/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546079093
Jul 17 03:22:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:32.255Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"11\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546080791
Jul 17 03:22:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:32.255Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"msh-tsgx05\", asset_id=\"67\", condition=\"Progressing\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"prometheus-pushgateway\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"unknown\"}" timestamp=1689546084068
Jul 17 03:22:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:32.275Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", phase=\"Running\", pod=\"tsg-traffic-engine-numa-0-firewall-76b5c5788d-2cs6q\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"99e7b7f0-307f-4274-a670-faf429f178f7\"}" timestamp=1689546084068
Jul 17 03:22:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:32.285Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"firewall\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Read\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689546081973
Jul 17 03:22:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:32.297Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db69\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546090141
Jul 17 03:22:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:32.307Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_limit_bytes\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"dign-client\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546082009
Jul 17 03:22:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:32.312Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"tmpfs\", name=\"var-lib-kubelet-pods-e186b759\\\\x2d1542\\\\x2d4733\\\\x2d9263\\\\x2d82c6ae7fed9e-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2db6gmg.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689546081015
Jul 17 03:22:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:32.314Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens8f0np0\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546079093
Jul 17 03:22:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:32.318Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_bytes_total\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"vethd59521a8\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546080791
Jul 17 03:22:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:32.331Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-a517240fe0ce891ae2d4ff19bb4be910cd0252f03c268bf31e201d97fa6e8b3e.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"scope\"}" timestamp=1689546086907
Jul 17 03:22:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:32.332Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_inodes_total\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/sys/fs/cgroup\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546072817
Jul 17 03:22:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:32.353Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_limit_bytes\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"lb-tcp-443\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546082009
Jul 17 03:22:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:32.356Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"112\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546080791
Jul 17 03:22:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:32.356Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_down_changes_total\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens1f0np0\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546079093
Jul 17 03:22:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:32.374Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:00-0000:00:01.1-0000:01:00.1-net-ens1f1np1.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689546082644
Jul 17 03:22:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:32.378Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_receive_switch_relay_errors_total\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_11\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546080791
Jul 17 03:22:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:32.379Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_drop_total\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens3f0np0\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546079093
Jul 17 03:22:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:32.384Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_rss\", asset=\"twa-tsgx03\", asset_id=\"95\", container=\"mrapm-device\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546075791
Jul 17 03:22:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:32.395Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_address_assign_type\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"cnibr0\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546080939
Jul 17 03:22:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:32.397Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"mlocate-updatedb.timer\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"timer\"}" timestamp=1689546086907
Jul 17 03:22:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:32.401Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"twa-tsgx03\", asset_id=\"95\", condition=\"false\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-traefik-crd-xvctc\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"037f9a49-f9c9-47d9-b6d3-eeb2a6dc570c\"}" timestamp=1689546090436
Jul 17 03:22:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:32.413Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-ens4f0np0.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689546082644
Jul 17 03:22:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:32.415Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_annotations\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"chart-values-traefik-crd\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546090436
Jul 17 03:22:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:32.421Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"125\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546080791
Jul 17 03:22:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:32.429Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"multipathd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"service\"}" timestamp=1689546081015
Jul 17 03:22:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:32.432Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546079093
Jul 17 03:22:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:32.435Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_compressed_total\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth15723f66\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546080791
Jul 17 03:22:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:32.525Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_id\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens4f1np1\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546079093
Jul 17 03:22:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:32.538Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:a0-0000:a0:03.1-0000:a1:00.0-infiniband-mlx5_6.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689546082644
Jul 17 03:22:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:32.549Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_created\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"localhost.node-password.k3s\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546090436
Jul 17 03:22:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:32.571Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_hwmon_temp_celsius\", asset=\"msh-tsgx05\", asset_id=\"67\", chip=\"0000:60:03_1_0000:63:00_2\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", sensor=\"temp1\", service=\"prometheus-node-exporter\"}" timestamp=1689546080939
Jul 17 03:22:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:32.585Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"121\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546079093
Jul 17 03:22:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:32.590Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-kernel-debug.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689546081015
Jul 17 03:22:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:32.594Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_files_free\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"shm\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/af52193df88507c0be48a18617e53a37938ce1fc646b8c36b30f9095e8fd086c/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546079093
Jul 17 03:22:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:32.606Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx05\", asset_id=\"99\", container=\"dign-server-dns\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Discard\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546077003
Jul 17 03:22:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:32.609Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_memory_used_dataset_bytes\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546086438
Jul 17 03:22:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:32.616Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"chronyd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"service\"}" timestamp=1689546082644
Jul 17 03:22:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:32.623Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_annotations\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", job_name=\"helm-delete-tsg-traffic-engine-numa-0\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546084068
Jul 17 03:22:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:32.641Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_info\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.tsg-traffic-engine-numa-1.v2\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546084068
Jul 17 03:22:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:32.645Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_spec_strategy_rollingupdate_max_unavailable\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"kubernetes-dashboard\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546079276
Jul 17 03:22:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:32.656Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_state_started\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"firewall\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", pod=\"tsg-traffic-engine-numa-0-firewall-99cddfc69-25wpt\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"abf27d03-a067-448d-81a8-11df769a2208\"}" timestamp=1689546079276
Jul 17 03:22:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:32.657Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_up_changes_total\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth7e541fcd\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546080791
Jul 17 03:22:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:32.662Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-abf27d03\\\\x2da067\\\\x2d448d\\\\x2d81a8\\\\x2d11df769a2208-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2dm8kps.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689546082644
Jul 17 03:22:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:32.674Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_data_transmitted_bytes_total\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_5\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546080791
Jul 17 03:22:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:32.675Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_resident_memory_bytes\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-udevd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689546082644
Jul 17 03:22:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:32.722Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_persistentvolume_labels\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", persistentvolume=\"pvc-a3534228-dbb7-4e18-b082-72065af86b7a\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546090436
Jul 17 03:22:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:32.759Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_max_fds\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"rtkit-daemon.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689546082644
Jul 17 03:22:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:32.864Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db243\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546086438
Jul 17 03:22:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:32.889Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db170\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546086438
Jul 17 03:22:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:32.923Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_NodeManager_ContainerLaunchDurationNumOps\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-Yarn-twa-dellr740-dt001-NodeManager\", endpoint_id=\"363\", module=\"DC-Yarn\", module_id=\"1\", name=\"NodeManagerMetrics\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546091354
Jul 17 03:22:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:32.943Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", phase=\"Running\", pod=\"dashboard-metrics-scraper-5b94897dc9-vgszb\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"4d0583db-52e0-420b-838b-92b5b61d0a0b\"}" timestamp=1689546090436
Jul 17 03:22:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:32.954Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_labels\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-0-firewall-686c8b6fbd\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546090436
Jul 17 03:22:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:32.958Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db219\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546076711
Jul 17 03:22:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:32.986Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"dign-client-s49j9\", project=\"TSG-OS\", project_id=\"7\", reason=\"NodeLost\", service=\"prometheus-kube-state-metrics\", uid=\"2096ff4a-92d2-47a7-9b70-293a6568cee4\"}" timestamp=1689546079276
Jul 17 03:22:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:33.042Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"dashboard-metrics-scraper\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Write\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546071961
Jul 17 03:22:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:33.062Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_oom_events_total\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"dign-server-web\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546068033
Jul 17 03:22:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:33.073Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_max_usage_bytes\", asset=\"twa-tsgx03\", asset_id=\"95\", container=\"lb-tcp-443\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546057470
Jul 17 03:22:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:33.115Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_packets_total\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens3f1np1\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546080791
Jul 17 03:22:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:33.132Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_cpu_seconds_total\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-pod345ff5d0_20c3_4ba4_bfcb_02d0c4070a47.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"slice\"}" timestamp=1689546082644
Jul 17 03:22:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:33.139Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_sector_writes_total\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/sys/fs/cgroup\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546072817
Jul 17 03:22:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:33.181Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"119\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546080791
Jul 17 03:22:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:33.198Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"user.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"slice\"}" timestamp=1689546082644
Jul 17 03:22:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:33.449Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_state_started\", asset=\"twa-tsgx05\", asset_id=\"99\", container=\"traefik\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"traefik-64f55bb67d-zfdn8\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"4984267b-c91e-4d03-b7d6-3defa3eedc21\"}" timestamp=1689546079894
Jul 17 03:22:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:33.505Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-81d6559a66a1e6f210b7e60851be216d5f122b3274df14270700da6f22f8e8c4.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"scope\"}" timestamp=1689546087691
Jul 17 03:22:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:33.528Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-bdd10b0d\\\\x2dcc33\\\\x2d4a4f\\\\x2d9622\\\\x2d89ebf267e6c3-volume\\\\x2dsubpaths-config\\\\x2dvolume-telegraf-1.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689546087691
Jul 17 03:22:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:33.704Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_status_succeeded\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", job_name=\"helm-install-traefik-crd\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546079894
Jul 17 03:22:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:33.720Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_labels\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"configmap-reloader-6c55bbc49f-tvvkl\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"5eafbc6c-7214-4748-a907-76b294823901\"}" timestamp=1689546079894
Jul 17 03:22:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:33.724Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_errors_received_total\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_8\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546080605
Jul 17 03:22:38 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:38.459Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_30_193_1_metric_majorCompactionOutputFileCount_25th_percentile\", asset=\"twa-dellr740-dt003\", asset_id=\"41\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt003-HregionServer\", endpoint_id=\"163\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"1\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546096623
Jul 17 03:22:38 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:38.683Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_frequency_min_hertz\", asset=\"twa-dellr740-dt003\", asset_id=\"41\", cpu=\"11\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"OLAP-Node-Exporter-twa-dellr740-dt003\", endpoint_id=\"261\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"hbase_master,clickhouse_data,kafka,gohangout,zookeeper,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546098473
Jul 17 03:22:39 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:39.288Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filefd_allocated\", asset=\"twa-dellr740-dt003\", asset_id=\"41\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"OLAP-Node-Exporter-twa-dellr740-dt003\", endpoint_id=\"261\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"hbase_master,clickhouse_data,kafka,gohangout,zookeeper,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546098473
Jul 17 03:22:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:41.283Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_next_gc_bytes\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546091647
Jul 17 03:22:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:41.302Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"twa-tsgx04\", asset_id=\"97\", condition=\"Progressing\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"mrzcpd-exporter\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"false\"}" timestamp=1689546100034
Jul 17 03:22:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:41.325Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"98\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546091647
Jul 17 03:22:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:41.361Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_daemonset_created\", asset=\"twa-tsgx04\", asset_id=\"97\", daemonset=\"dign-server\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546100034
Jul 17 03:22:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:41.377Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"114\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546091647
Jul 17 03:22:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:41.500Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_size_bytes\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"shm\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/47a07edd11ecc492029afdaa1b6e5cbd6e3a32d5f3c9d5fa234e8487313888d9/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546091647
Jul 17 03:22:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:41.664Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"twa-tsgx04\", asset_id=\"97\", condition=\"unknown\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"local-path-provisioner-957fdf8bc-t9pfz\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"5b3f6272-57d2-4b0d-83be-046bdee2c23b\"}" timestamp=1689546100034
Jul 17 03:22:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:41.701Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"10\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546091647
Jul 17 03:22:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:41.704Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"tmpfs\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/data/var/lib/kubelet/pods/9f402370-116a-4e69-9fa9-b0192f6cd502/volumes/kubernetes.io~projected/kube-api-access-7kf5v\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546091647
Jul 17 03:22:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:41.746Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_maxCompactionQueueSize\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"assessment_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"7b09e8bea901f44621baa2a2f9e32384\", table=\"index_time_assessment_hos_bucket\", type=\"index\"}" timestamp=1689546099757
Jul 17 03:22:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:41.794Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_appendCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"proxy_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"04686c29bda3691b5a5780ae64a5b385\", table=\"index_filename_proxy_hos_bucket\", type=\"index\"}" timestamp=1689546099757
Jul 17 03:22:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:41.818Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_compactionsCompletedCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"troubleshooting_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"2bb40fcd716437b4939ab03c58064dd2\", table=\"index_time_troubleshooting_hos_bucket\", type=\"index\"}" timestamp=1689546099757
Jul 17 03:22:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:41.865Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_numBytesCompactedCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"rtp_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"fa6c8d612c8d2911ad7f19b8c951ea52\", table=\"index_partfile_rtp_hos_bucket\", type=\"index\"}" timestamp=1689546099757
Jul 17 03:22:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:41.988Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"twa-tsgx04\", asset_id=\"97\", condition=\"Progressing\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"dashboard-metrics-scraper\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"true\"}" timestamp=1689546100034
Jul 17 03:22:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:42.007Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_mtu_bytes\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vetha33ccb73\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546091647
Jul 17 03:22:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:42.029Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"95\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546091647
Jul 17 03:22:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:42.157Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"98\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546091647
Jul 17 03:22:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:42.201Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_maxStoreFileAge\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"04f9e2d9401ea405fe3edb2a67813afe\", table=\"troubleshooting_hos_bucket\", type=\"data\"}" timestamp=1689546099757
Jul 17 03:22:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:42.215Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_blockCacheDeleteFamilyBloomMissCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Server\"}" timestamp=1689546099757
Jul 17 03:22:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:42.240Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"75\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546091647
Jul 17 03:22:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:42.250Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_10_194_1_metric_compactionTime_99th_percentile\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546099757
Jul 17 03:22:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:42.264Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", fstype=\"ext4\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/data/var/lib/kubelet/pods/9ad11045-e0bd-4481-9c2d-4c81a67475ee/volume-subpaths/config-volume/telegraf-security/2\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546091647
Jul 17 03:22:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:42.455Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_numReferenceFiles\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"proxy_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"d50358d51b185086f76e71418eb6ac7d\", table=\"index_time_proxy_hos_bucket\", type=\"index\"}" timestamp=1689546099757
Jul 17 03:22:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:42.472Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_10_194_1_metric_flushMemstoreSize_min\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546099757
Jul 17 03:22:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:42.477Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_system_table_bucket_meta_metric_flushMemstoreSize_98th_percentile\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546099757
Jul 17 03:22:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:42.493Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_minStoreFileAge\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", namespace=\"SYSTEM\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"e1edba3b81b21a74ea026f21ba7bed91\", table=\"LOG\", type=\"data\"}" timestamp=1689546099757
Jul 17 03:22:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:42.575Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_heap_inuse_bytes\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546094838
Jul 17 03:22:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:42.785Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", cpu=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546096242
Jul 17 03:22:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:43.020Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_packets_received_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"mlx5_2\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546096242
Jul 17 03:22:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:43.023Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"pcap-tsgx01\", asset_id=\"71\", condition=\"unknown\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-systemd-exporter-cbljh\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"da14c78d-f512-4a0f-b73a-808e63c92307\"}" timestamp=1689546095154
Jul 17 03:22:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:43.034Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_compressed_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth54027b61\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546096242
Jul 17 03:22:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:43.035Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_labels\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"chart-values-traefik\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546095154
Jul 17 03:22:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:43.055Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db66\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546094838
Jul 17 03:22:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:43.406Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-tsg-traffic-engine-numa-0-mmwxx\", project=\"TSG-OS\", project_id=\"7\", reason=\"NodeLost\", service=\"prometheus-kube-state-metrics\", uid=\"ae5d5bd3-3f8a-4eac-9409-8356a4aef052\"}" timestamp=1689546095154
Jul 17 03:22:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:43.418Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_netstat_Icmp6_OutMsgs\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546096242
Jul 17 03:22:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:43.478Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"scrape_duration_seconds\", asset=\"msh-dellr740-olap005\", asset_id=\"19\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Hadoop-msh-dellr740-olap005-DataNode\", endpoint_id=\"275\", instance=\"192.168.20.225:9906\", job=\"275\", module=\"NC-Hadoop\", module_id=\"37\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546103036
Jul 17 03:22:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:43.566Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2did-scsi\\\\x2dSDELL_PERC_H345_Adp_00eaf83a825e7c2c2c00558284f0a7ce\\\\x2dpart1.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689546100671
Jul 17 03:22:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:43.624Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"twa-tsgx02\", asset_id=\"93\", cpu=\"116\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx02\", endpoint_id=\"445\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546102426
Jul 17 03:22:47 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:47.514Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_fsChecksumFailureCount\", asset=\"pcap-dellr740-dt003\", asset_id=\"35\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt003-Hmaster\", endpoint_id=\"387\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"IO\"}" timestamp=1689546106370
Jul 17 03:22:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:49.166Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_lastMajorCompactionAge\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"session_record_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"d0ac010b6fb426ff7fbaccfe3067b92d\", table=\"index_time_session_record_hos_bucket\", type=\"index\"}" timestamp=1689546107438
Jul 17 03:22:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:49.182Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_hbase_table_rsgroup_metric_scanTime_min\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546107438
Jul 17 03:22:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:49.208Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_1_metric_tableSize\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546107438
Jul 17 03:22:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:49.223Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_maxFlushQueueSize\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"assessment_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"a7a5c13a400401472e2d36778e8ca3a1\", table=\"index_time_assessment_hos_bucket\", type=\"index\"}" timestamp=1689546107438
Jul 17 03:22:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:49.306Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_session_record_hos_bucket_metric_compactionInputFileCount_95th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546107438
Jul 17 03:22:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:49.319Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_replicaid\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"session_record_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"d7b6a425d82c7496478be02811a70285\", table=\"index_partfile_session_record_hos_bucket\", type=\"index\"}" timestamp=1689546107438
Jul 17 03:22:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:49.360Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_4_metric_compactionOutputSize_25th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546107438
Jul 17 03:22:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:49.424Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_compactionsCompletedCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"rtp_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"87a4e8e69bf9d628687a1fe7e7d6467c\", table=\"index_time_rtp_hos_bucket\", type=\"index\"}" timestamp=1689546107438
Jul 17 03:22:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:49.694Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_speed_bytes\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth23566533\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546101797
Jul 17 03:22:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:49.737Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_lastMajorCompactionAge\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"rtp_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"b242966859aa9721f3ac6e8b17d51a4d\", table=\"index_filename_rtp_hos_bucket\", type=\"index\"}" timestamp=1689546107438
Jul 17 03:22:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:49.753Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_session_record_hos_bucket_metric_putBatchTime_max\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546107438
Jul 17 03:22:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:49.784Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-netns-cni\\\\x2d208839f0\\\\x2dcf2c\\\\x2d13e7\\\\x2d4d95\\\\x2d6092b9dffd9d.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689546101057
Jul 17 03:22:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:49.863Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-node-exporter-bz7wf\", project=\"TSG-OS\", project_id=\"7\", reason=\"NodeAffinity\", service=\"prometheus-kube-state-metrics\", uid=\"d58b8827-5397-48fd-9ce6-3976a10dcb09\"}" timestamp=1689546099668
Jul 17 03:22:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:49.939Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_labels\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"traefik-64f55bb67d\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546099668
Jul 17 03:22:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:50.036Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_virtual_memory_bytes\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"watchdog.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689546106554
Jul 17 03:22:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:50.073Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-20025742b4f87b7ff1b63ca328d232a8493992ad6cc32f04292c37fe5dc27577.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"scope\"}" timestamp=1689546106554
Jul 17 03:22:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:50.115Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"43\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546106709
Jul 17 03:22:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:50.126Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db144\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546106625
Jul 17 03:22:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:50.507Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"108\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546106709
Jul 17 03:22:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:50.610Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_virtual_memory_max_bytes\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"crond.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689546106554
Jul 17 03:22:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:50.700Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_inodes_free\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/96a9f56f8880670d84ba1753f731c63c62abdde31dae3ba4a5cb169300591055/shm\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546099758
Jul 17 03:22:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:50.785Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_inodes_free\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"overlay_0-307\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546099758
Jul 17 03:22:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:50.786Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db58\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546106625
Jul 17 03:22:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:50.806Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_limit_bytes\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"overlay_0-372\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546099758
Jul 17 03:22:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:50.818Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"twa-tsgx10\", asset_id=\"109\", cpu=\"102\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546101797
Jul 17 03:22:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:50.843Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_sector_reads_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/a379a67869d57aedcb0526bd2cb5e1a6fe7faed6994dcda2a2fc2426d6838604/shm\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546099758
Jul 17 03:22:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:50.845Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_hwmon_temp_celsius\", asset=\"twa-tsgx08\", asset_id=\"105\", chip=\"pci0000:00_0000:00:19_3\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", sensor=\"temp7\", service=\"prometheus-node-exporter\"}" timestamp=1689546106709
Jul 17 03:22:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:50.871Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"twa-tsgx08\", asset_id=\"105\", container=\"prometheus-alertmanager-configmap-reload\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", failure_type=\"pgmajfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"hierarchy\"}" timestamp=1689546106275
Jul 17 03:22:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:50.884Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"overlay\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-f949fb18bd59c192bd47b82c61c74ebc34fc8a8d776675553ef040c7db73dcc9-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689546101057
Jul 17 03:22:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:50.899Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_rss\", asset=\"twa-tsgx08\", asset_id=\"105\", container=\"prometheus-server\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546090921
Jul 17 03:22:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:50.900Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"87\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546106709
Jul 17 03:22:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:50.930Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-1617bf89\\\\x2d68ca\\\\x2d42e6\\\\x2d8a6e\\\\x2da79f76bf18ce-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2dhjpfx.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689546106554
Jul 17 03:22:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:50.935Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_frequency_min_hertz\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"85\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546106709
Jul 17 03:22:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:50.976Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_io_time_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"overlay_0-240\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546099758
Jul 17 03:22:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:51.006Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_usage_bytes\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/data/var/lib/kubelet/pods/132d373d-8755-45d4-97c7-04927f0acd43/volumes/kubernetes.io~projected/kube-api-access-kncnn\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546099758
Jul 17 03:22:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:51.044Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_write_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/var/lib/kubelet/pods/1617bf89-68ca-42e6-8a6e-a79f76bf18ce/volumes/kubernetes.io~projected/kube-api-access-hjpfx\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546099758
Jul 17 03:22:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:51.077Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_xfs_vnode_reclaim_total\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"sdb1\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-olap003\", endpoint_id=\"235\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"yarn_resourcemanager,zookeeper,nacos,kafka,spark,druid_data,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546109713
Jul 17 03:22:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:51.117Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_net_dev_group\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens3f0np0\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546106709
Jul 17 03:22:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:51.132Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db79\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546107556
Jul 17 03:22:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:51.138Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_swap\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"telegraf-security\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689546087445
Jul 17 03:22:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:51.175Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_errors_received_total\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_11\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546101797
Jul 17 03:22:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:51.194Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_compressed_total\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethec314a7d\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546101797
Jul 17 03:22:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:51.205Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_address_available\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_endpoint=\"prometheus-pushgateway\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546099668
Jul 17 03:22:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:51.219Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", cpu=\"8\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-olap003\", endpoint_id=\"235\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"yarn_resourcemanager,zookeeper,nacos,kafka,spark,druid_data,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546109713
Jul 17 03:22:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:51.219Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_status_start_time\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", job_name=\"helm-install-prometheus\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546099668
Jul 17 03:22:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:51.235Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_waiting\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"helm\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-traefik-crd-h4dtp\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"440a9824-0943-4c19-88b9-06b635f437bf\"}" timestamp=1689546099668
Jul 17 03:22:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:51.256Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_scrape_collector_success\", asset=\"twa-tsgx08\", asset_id=\"105\", collector=\"ipvs\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546106709
Jul 17 03:22:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:51.281Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"87\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546106709
Jul 17 03:22:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:51.321Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_io_time_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/a379a67869d57aedcb0526bd2cb5e1a6fe7faed6994dcda2a2fc2426d6838604/shm\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546099758
Jul 17 03:22:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:51.340Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"twa-tsgx08\", asset_id=\"105\", container=\"metrics-server\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"running\"}" timestamp=1689546092684
Jul 17 03:22:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:51.408Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_reads_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689546104385
Jul 17 03:22:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:51.425Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-ask-password-wall.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"service\"}" timestamp=1689546106554
Jul 17 03:22:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:51.478Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"scrape_samples_post_metric_relabeling\", asset=\"twa-dellr740-dt002\", asset_id=\"39\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-Hadoop-twa-dellr740-dt002-DataNode\", endpoint_id=\"197\", instance=\"192.168.30.194:9906\", job=\"197\", module=\"DC-Hadoop\", module_id=\"53\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546110856
Jul 17 03:22:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:51.499Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_annotations\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", job_name=\"helm-install-tsg-traffic-engine-numa-1\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546101568
Jul 17 03:22:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:51.500Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"jvm_gc_pause_seconds_max\", action=\"end of minor GC\", application=\"bifang_api\", asset=\"pcap-kvm-cm001\", asset_id=\"45\", cause=\"Allocation Failure\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"CM-API-pcap-kvm-cm001\", endpoint_id=\"285\", module=\"CM-API\", module_id=\"55\", nz_agent_id=\"3\", parent_asset=\"pcap-dellr740-pve001\", parent_asset_id=\"43\", project=\"Central-Management\", project_id=\"5\"}" timestamp=1689546110876
Jul 17 03:22:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:51.554Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"cache_misses_total\", application=\"bifang_api\", asset=\"pcap-kvm-cm001\", asset_id=\"45\", cache=\"SESSION\", cacheManager=\"cacheManager\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"CM-API-pcap-kvm-cm001\", endpoint_id=\"285\", module=\"CM-API\", module_id=\"55\", name=\"SESSION\", nz_agent_id=\"3\", parent_asset=\"pcap-dellr740-pve001\", parent_asset_id=\"43\", project=\"Central-Management\", project_id=\"5\", reason=\"notFound\"}" timestamp=1689546110876
Jul 17 03:22:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:51.761Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_inodes_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/run\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546099758
Jul 17 03:22:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:51.820Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"127\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546106709
Jul 17 03:22:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:52.501Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_threads_max\", asset=\"pcap-tsgx11\", asset_id=\"115\", container=\"monit-stream\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546104804
Jul 17 03:22:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:52.517Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"pcap-tsgx11\", asset_id=\"115\", condition=\"Available\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"kubernetes-dashboard\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"true\"}" timestamp=1689546104478
Jul 17 03:22:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:52.560Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_info\", asset=\"pcap-tsgx11\", asset_id=\"115\", created_by_kind=\"DaemonSet\", created_by_name=\"svclb-traefik-644fb0c0\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", host_ip=\"10.10.10.171\", host_network=\"false\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", node=\"localhost\", nz_agent_id=\"3\", pod=\"svclb-traefik-644fb0c0-7cq8s\", pod_ip=\"10.42.0.27\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"ea6cc882-864f-480f-b3a6-4af3bfdaa023\"}" timestamp=1689546104478
Jul 17 03:22:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:52.598Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_info\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.tsg-traffic-engine-numa-1.v2\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546104478
Jul 17 03:22:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:52.671Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_io_time_seconds_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda5\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546105967
Jul 17 03:22:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:52.674Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2dpath-pci\\\\x2d0000:81:00.0\\\\x2dscsi\\\\x2d0:2:0:0\\\\x2dpart3.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689546105496
Jul 17 03:22:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:52.710Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_restart_policy\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-pushgateway-bd476bcf9-8t7vj\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", type=\"Always\", uid=\"f2baf82c-b91b-46ec-8a59-8f55f1b86285\"}" timestamp=1689546104478
Jul 17 03:22:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:52.746Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_compressed_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth561c9aba\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546109337
Jul 17 03:22:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:52.759Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_drop_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens7f0np0\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546109337
Jul 17 03:22:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:52.798Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", cpu=\"1\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546109337
Jul 17 03:22:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:52.841Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_local_link_integrity_errors_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"mlx5_4\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546109337
Jul 17 03:22:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:52.939Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2dpartlabel-TSG\\\\x2dOS\\\\x2dDATA.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689546105496
Jul 17 03:22:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:53.173Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"msh-tsgx08\", asset_id=\"65\", cpu=\"81\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546108328
Jul 17 03:22:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:53.185Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db199\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546110108
Jul 17 03:22:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:53.191Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", cpu=\"125\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546109337
Jul 17 03:22:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:53.203Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_created\", asset=\"msh-tsgx08\", asset_id=\"65\", configmap=\"multus-cni-config\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546108950
Jul 17 03:22:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:53.206Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-ttyS3.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689546105496
Jul 17 03:22:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:53.258Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-prometheus-k5mgl\", project=\"TSG-OS\", project_id=\"7\", reason=\"Shutdown\", service=\"prometheus-kube-state-metrics\", uid=\"75952f28-65f8-4ba3-afae-3c693a5ebfb7\"}" timestamp=1689546108663
Jul 17 03:22:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:53.266Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"msh-tsgx08\", asset_id=\"65\", cpu=\"123\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546108328
Jul 17 03:22:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:53.283Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_created\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"systemd-exporter-7f549d9485\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546108663
Jul 17 03:22:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:53.294Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"time-sync.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"target\"}" timestamp=1689546112135
Jul 17 03:22:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:53.414Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_packets_total\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"cadvisor\", interface=\"ens4f3np3\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546093388
Jul 17 03:22:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:53.536Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"msh-tsgx08\", asset_id=\"65\", condition=\"true\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-systemd-exporter-k9pkr\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"8db4e66c-b88f-4e1e-a738-4f13414c9d3b\"}" timestamp=1689546108950
Jul 17 03:22:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:53.725Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_rss\", asset=\"pcap-tsgx11\", asset_id=\"115\", container=\"metrics\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546103349
Jul 17 03:22:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:53.770Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_created\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"coredns\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546104478
Jul 17 03:22:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:53.867Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sshd-keygen@rsa.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"service\"}" timestamp=1689546105496
Jul 17 03:22:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:53.952Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:00-0000:00:01.1-0000:01:00.0-infiniband-mlx5_0.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689546105496
Jul 17 03:22:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:54.111Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"clevis-luks-askpass.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"service\"}" timestamp=1689546105496
Jul 17 03:22:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:54.236Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_address_assign_type\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth8ec198c4\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546103391
Jul 17 03:22:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:54.287Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_speed_bytes\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth068249f9\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546103391
Jul 17 03:22:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:54.425Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"msh-tsgx08\", asset_id=\"65\", condition=\"unknown\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"configmap-reloader-6c55bbc49f-llzs5\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"8f360efb-8f56-4f00-abc0-fca4bf50282a\"}" timestamp=1689546108950
Jul 17 03:22:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:54.472Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"59\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546108328
Jul 17 03:22:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:54.518Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx08\", asset_id=\"65\", cpu=\"104\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546108328
Jul 17 03:22:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:54.519Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_up_changes_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"bond0\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546099409
Jul 17 03:22:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:54.741Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_down_changes_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"vethd26d000c\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546099409
Jul 17 03:22:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:54.878Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_info\", asset=\"msh-tsgx08\", asset_id=\"65\", configmap=\"shaping-tsg-traffic-engine-numa-0\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546108950
Jul 17 03:22:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:54.893Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_name_assign_type\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"eno2\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546103391
Jul 17 03:22:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:54.920Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"msh-tsgx12\", asset_id=\"113\", cpu=\"107\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546103391
Jul 17 03:22:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:54.926Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_ready\", asset=\"pcap-tsgx07\", asset_id=\"85\", container=\"metrics-server\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"metrics-server-54dc485875-jkvd8\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"0a0a8b78-3fe6-4b35-93e0-3e1bdd94459c\"}" timestamp=1689546102058
Jul 17 03:22:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:54.944Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-34d06245e6306777e62cac2af62177d46a07b3092c0f76399f1329200a6c476d.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"scope\"}" timestamp=1689546111179
Jul 17 03:22:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:54.981Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dm-event.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"service\"}" timestamp=1689546111179
Jul 17 03:22:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:54.984Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_packets_dropped_total\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"cadvisor\", interface=\"eno33np0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546093388
Jul 17 03:22:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:55.006Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"msh-tsgx08\", asset_id=\"65\", container=\"lb-tcp-443\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"running\"}" timestamp=1689546093699
Jul 17 03:22:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:55.015Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"mlocate-updatedb.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"service\"}" timestamp=1689546111179
Jul 17 03:22:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:55.043Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_fifo_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth55feb6e8\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546099409
Jul 17 03:22:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:55.206Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_gc_sys_bytes\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546098659
Jul 17 03:22:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:55.215Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_labels\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"metrics-server\", uid=\"31e112e6-f92d-41a2-b1ae-2ef2cf87ad07\"}" timestamp=1689546102058
Jul 17 03:22:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:55.487Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"msh-tsgx12\", asset_id=\"113\", container=\"firewall\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"cadvisor\", failure_type=\"pgmajfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"hierarchy\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689546094871
Jul 17 03:22:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:55.588Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_limit_bytes\", asset=\"msh-tsgx12\", asset_id=\"113\", container=\"telegraf-security\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689546100589
Jul 17 03:22:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:55.616Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_persistentvolumeclaim_status_phase\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", persistentvolumeclaim=\"prometheus-alertmanager\", phase=\"Bound\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546108663
Jul 17 03:22:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:55.638Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"msh-tsgx12\", asset_id=\"113\", cpu=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546103391
Jul 17 03:22:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:55.656Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cpu=\"105\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546099409
Jul 17 03:22:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:55.681Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_up_changes_total\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth4e8b65d2\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546103391
Jul 17 03:22:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:55.701Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx08\", asset_id=\"65\", cpu=\"49\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546108328
Jul 17 03:22:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:55.715Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"auth-rpcgss-module.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"service\"}" timestamp=1689546111179
Jul 17 03:22:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:55.821Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-netns-cni\\\\x2daab1e3bd\\\\x2de08f\\\\x2d2476\\\\x2d0bd8\\\\x2df30aabfb1d86.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689546111179
Jul 17 03:22:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:55.960Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sysstat-summary.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"service\"}" timestamp=1689546100884
Jul 17 03:22:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:56.440Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-aedc03bd4ea68d01b02ba967ebd416aa1a28bc9d071f1cf4154e08980e554c2b-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689546103372
Jul 17 03:22:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:56.556Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_daemonset_created\", asset=\"pcap-tsgx03\", asset_id=\"73\", daemonset=\"dign-client\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546113389
Jul 17 03:22:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:56.669Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_name_assign_type\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth883fff26\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546099409
Jul 17 03:22:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:56.706Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas_ready\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"local-path-provisioner\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546102058
Jul 17 03:22:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:56.710Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_last_terminated_reason\", asset=\"pcap-tsgx03\", asset_id=\"73\", container=\"dashboard-metrics-scraper\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", pod=\"dashboard-metrics-scraper-5b94897dc9-pkp4g\", project=\"TSG-OS\", project_id=\"7\", reason=\"Unknown\", service=\"prometheus-kube-state-metrics\", uid=\"99474c52-8652-4755-b917-62a7ae1a3d6c\"}" timestamp=1689546113389
Jul 17 03:22:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:56.764Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_cache\", asset=\"pcap-tsgx03\", asset_id=\"73\", container=\"prometheus-pushgateway\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546100019
Jul 17 03:22:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:56.770Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_labels\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.tsg-traffic-engine-numa-0.v3\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546102058
Jul 17 03:22:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:56.820Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"72\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546099409
Jul 17 03:22:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:57.159Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_oom_events_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", container=\"dashboard-metrics-scraper\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546098136
Jul 17 03:22:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:57.391Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", container=\"dign-server-web\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"running\"}" timestamp=1689546102559
Jul 17 03:22:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:57.483Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_limit_bytes\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689546111563
Jul 17 03:22:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:57.507Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"traefik-64f55bb67d-ctv58\", project=\"TSG-OS\", project_id=\"7\", reason=\"UnexpectedAdmissionError\", service=\"prometheus-kube-state-metrics\", uid=\"1546be34-e5d0-4c1b-b86b-be67b4c66d65\"}" timestamp=1689546113389
Jul 17 03:22:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:57.512Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_ready\", asset=\"pcap-tsgx03\", asset_id=\"73\", container=\"mrapm-device\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"mrzcpd-exporter-5cf7f967cc-q98g4\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"900223f6-7495-4cd9-b023-184522fd9a95\"}" timestamp=1689546113389
Jul 17 03:22:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:57.538Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"120\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546105883
Jul 17 03:22:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:57.545Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_memory_FilePmdMapped_bytes\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546105883
Jul 17 03:22:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:57.667Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_created\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.tsg-traffic-engine-numa-1.v3\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546113389
Jul 17 03:22:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:57.675Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_labels\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"kubernetes-dashboard-66dd5b456\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546113389
Jul 17 03:22:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:57.697Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-3050ccdaf8165c965954b1326690caeef6dbd5ccaa63155fbfe7b6a725b3e8bb.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"scope\"}" timestamp=1689546103372
Jul 17 03:22:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:57.830Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens1f0np0\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546105883
Jul 17 03:22:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:57.896Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_ready_replicas\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-0-firewall-5bf9999fcf\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546113389
Jul 17 03:22:58 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:58.482Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_link_downed_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"mlx5_3\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546105883
Jul 17 03:22:58 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:58.577Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-rfkill.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689546103372
Jul 17 03:22:58 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:58.604Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_frequency_min_hertz\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", cpu=\"4\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"OLAP-Node-Exporter-pcap-dellr740-dt001\", endpoint_id=\"251\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546117557
Jul 17 03:22:58 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:58.849Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"k3s-orphaned-pods-dir-clean.timer\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"timer\"}" timestamp=1689546103372
Jul 17 03:22:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:59.195Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"msh-dellr740-dt003\", asset_id=\"9\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-dt003\", endpoint_id=\"229\", module=\"OLAP-Node-Exporter\", module_id=\"51\", name=\"4\", nz_agent_id=\"3\", olap_node_exporter=\"hbase_master,clickhouse_data,kafka,gohangout,zookeeper,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\", type=\"Processor\"}" timestamp=1689546117407
Jul 17 03:22:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:59.217Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_scrape_collector_success\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", collector=\"zfs\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"OLAP-Node-Exporter-pcap-dellr740-dt001\", endpoint_id=\"251\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546117557
Jul 17 03:22:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:59.649Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth874693b4\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546109083
Jul 17 03:22:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:59.663Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_speed_bytes\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth366ee37d\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546109083
Jul 17 03:22:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:59.681Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"27\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546109083
Jul 17 03:22:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:59.811Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"49\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546109083
Jul 17 03:22:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:59.939Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db23\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546105071
Jul 17 03:22:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:22:59.954Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-d566c081f7ecaa1ce639fc83148816c9c9c1c90e3d18762e07c5b16cc00c5055.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"scope\"}" timestamp=1689546104879
Jul 17 03:23:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:00.518Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_constraint_errors_transmitted_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_8\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546109083
Jul 17 03:23:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:00.610Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"eno2\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546109083
Jul 17 03:23:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:00.611Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_packets_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens4f0np0\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546109083
Jul 17 03:23:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:00.721Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db0\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546105071
Jul 17 03:23:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:00.738Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"initrd-fs.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"target\"}" timestamp=1689546104879
Jul 17 03:23:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:02.417Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_node_status_condition\", asset=\"msh-tsgx02\", asset_id=\"63\", condition=\"DiskPressure\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", node=\"localhost\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"false\"}" timestamp=1689546111412
Jul 17 03:23:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:02.774Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx02\", asset_id=\"63\", cpu=\"47\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546118318
Jul 17 03:23:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:02.794Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_netstat_Icmp_OutMsgs\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546118318
Jul 17 03:23:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:02.798Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx02\", asset_id=\"63\", cpu=\"82\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546118318
Jul 17 03:23:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:02.959Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_rss\", asset=\"msh-tsgx02\", asset_id=\"63\", container=\"prometheus-alertmanager\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546111212
Jul 17 03:23:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:02.990Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_multicast_total\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens4f1np1\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546118318
Jul 17 03:23:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:03.019Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_protocol_type\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens7f1np1\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546118318
Jul 17 03:23:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:03.079Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"msh-tsgx02\", asset_id=\"63\", cpu=\"122\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546118318
Jul 17 03:23:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:03.114Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"prometheus_tsdb_head_series\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", instance=\"10.10.20.159:10090\", type=\"global\"}" timestamp=1689546122990
Jul 17 03:23:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:03.143Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", cpu=\"0\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-dt001\", endpoint_id=\"225\", mode=\"nice\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546122987
Jul 17 03:23:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:03.182Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_annotations\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", uid=\"41d38bc8-e6f4-4e43-8336-25db4b6d60ba\"}" timestamp=1689546111412
Jul 17 03:23:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:03.454Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_rss\", asset=\"msh-tsgx02\", asset_id=\"63\", container=\"prometheus-pushgateway\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546108947
Jul 17 03:23:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:03.529Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_namespace_status_phase\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Active\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546111412
Jul 17 03:23:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:03.534Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"nfs-utils.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689546108447
Jul 17 03:23:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:03.553Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sssd-kcm.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689546108447
Jul 17 03:23:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:03.574Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-ens6f0np0.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689546108447
Jul 17 03:23:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:04.410Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"net_conntrack_dialer_conn_failed_total\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", dialer_name=\"481\", instance=\"10.10.20.159:10090\", reason=\"unknown\", type=\"global\"}" timestamp=1689546122990
Jul 17 03:23:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:04.442Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"prometheus_http_response_size_bytes_sum\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", handler=\"/-/healthy\", instance=\"10.10.20.159:10090\", type=\"global\"}" timestamp=1689546122990
Jul 17 03:23:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:04.580Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"prometheus_target_scrape_pool_targets\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", instance=\"10.10.20.159:10090\", scrape_job=\"211\", type=\"global\"}" timestamp=1689546122990
Jul 17 03:23:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:04.734Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"net_conntrack_dialer_conn_closed_total\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", dialer_name=\"asset_ping_115\", instance=\"10.10.20.159:10090\", type=\"global\"}" timestamp=1689546122990
Jul 17 03:23:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:06.945Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cpu=\"55\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546123910
Jul 17 03:23:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:07.226Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"ftx_pkts_total\", app=\"sapp4-1\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"nf_1_fw\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"mrzcpd-exporter\", serviceFunctionIndex=\"1\"}" timestamp=1689546118833
Jul 17 03:23:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:07.266Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"phy_rx_missed_total\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", dev=\"virtio_dign_s\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"mrzcpd-exporter\"}" timestamp=1689546118833
Jul 17 03:23:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:07.291Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_max_fds\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-journald.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689546113582
Jul 17 03:23:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:07.646Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_errs_total\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens4f1np1\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546123910
Jul 17 03:23:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:07.733Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cpupower.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"service\"}" timestamp=1689546113582
Jul 17 03:23:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:10.239Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_lastMajorCompactionAge\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"proxy_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"3f3a8f4c87c05c3707c187b169923a9f\", table=\"index_filename_proxy_hos_bucket\", type=\"index\"}" timestamp=1689546128804
Jul 17 03:23:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:10.483Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Bulkload_count\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Server\"}" timestamp=1689546128804
Jul 17 03:23:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:10.519Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_CheckAndPut_75th_percentile\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Server\"}" timestamp=1689546128804
Jul 17 03:23:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:10.648Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_maxFlushQueueSize\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"proxy_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"d74bb32ffdc6517c1bcb927023c29bee\", table=\"index_partfile_proxy_hos_bucket\", type=\"index\"}" timestamp=1689546128804
Jul 17 03:23:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:10.679Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_TotalCallTime_75th_percentile\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"IPC\"}" timestamp=1689546128804
Jul 17 03:23:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:11.370Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_avgStoreFileAge\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"4a09c716ddd98eca9c8e3970bdc0f848\", table=\"assessment_hos_bucket\", type=\"data\"}" timestamp=1689546128804
Jul 17 03:23:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:11.451Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_194_3_metric_deleteTime_mean\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546130148
Jul 17 03:23:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:11.742Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_replicaid\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"rtp_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"facb2c425102ca9759669be822cbcd05\", table=\"index_time_rtp_hos_bucket\", type=\"index\"}" timestamp=1689546130148
Jul 17 03:23:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:12.256Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_STATS_metric_appendTime_median\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546130148
Jul 17 03:23:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:12.280Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_alloc_bytes_total\", asset=\"twa-dellr740-dt002\", asset_id=\"39\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"OLAP-Node-Exporter-twa-dellr740-dt002\", endpoint_id=\"259\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546132056
Jul 17 03:23:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:12.301Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_MUTEX_metric_scanSize_min\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546130148
Jul 17 03:23:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:12.332Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_3_metric_incrementTime_mean\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546130148
Jul 17 03:23:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:12.382Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_tsg_table_report_result_metric_compactionOutputSize_max\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546130148
Jul 17 03:23:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:12.514Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_scanCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"session_record_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"b294aae0b11a6abdccfdd6fee84f700d\", table=\"index_time_session_record_hos_bucket\", type=\"index\"}" timestamp=1689546130148
Jul 17 03:23:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:12.627Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_storeFileSize\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"session_record_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"64ac4b5864b8b0132c1e942230dfc553\", table=\"index_time_session_record_hos_bucket\", type=\"index\"}" timestamp=1689546130148
Jul 17 03:23:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:12.645Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_session_record_hos_bucket_metric_flushTime_75th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546130148
Jul 17 03:23:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:12.659Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_4_metric_compactionInputFileCount_max\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546130148
Jul 17 03:23:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:12.685Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_CheckAndPut_max\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Server\"}" timestamp=1689546130148
Jul 17 03:23:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:12.723Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_STATS_metric_flushTime_99_9th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546130148
Jul 17 03:23:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:12.807Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_4_metric_deleteBatchTime_num_ops\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546130148
Jul 17 03:23:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:12.826Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_maxFlushQueueSize\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"session_record_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"6dacfb7337a3da49e28f46ddfef67879\", table=\"index_partfile_session_record_hos_bucket\", type=\"index\"}" timestamp=1689546130148
Jul 17 03:23:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:12.827Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_3_metric_appendTime_99_9th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546130148
Jul 17 03:23:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:12.846Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_memStoreSize\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"proxy_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"bfcd386716bdc9d91798aa0de532a51e\", table=\"index_time_proxy_hos_bucket\", type=\"index\"}" timestamp=1689546130148
Jul 17 03:23:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:12.943Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_storeCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"knowledge_base_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"1ee9566a08bea50a95dd060f332950cb\", table=\"index_partfile_knowledge_base_hos_bucket\", type=\"index\"}" timestamp=1689546130148
Jul 17 03:23:13 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:13.058Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_session_record_hos_bucket_metric_majorCompactionTime_num_ops\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546130148
Jul 17 03:23:13 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:13.075Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_STATS_metric_flushMemstoreSize_min\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546130148
Jul 17 03:23:13 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:13.089Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_193_1_metric_compactionOutputSize_95th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546130148
Jul 17 03:23:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:15.299Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_info\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", job_name=\"helm-install-configmap-reloader\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546128076
Jul 17 03:23:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:15.736Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_observed_generation\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"traefik\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546128076
Jul 17 03:23:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:15.850Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-e026c1a65633a8a9f1f06fe4c43008960cbf64ced235e8292b3b355c4b39b3e2-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689546127046
Jul 17 03:23:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:15.858Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-f38365d555dc90ba8717f695b5b55dcbd308488f5bcf7e13bdc929b9c72daeb1.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"scope\"}" timestamp=1689546127046
Jul 17 03:23:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:16.326Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2dpartlabel-ONIE\\\\x2dBOOT.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689546127046
Jul 17 03:23:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:16.499Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2dlabel-EFI\\\\x5cx20System.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689546127046
Jul 17 03:23:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:16.567Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_daemonset_status_number_available\", asset=\"twa-tsgx07\", asset_id=\"103\", daemonset=\"dign-client\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546128076
Jul 17 03:23:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:19.386Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-netns-cni\\\\x2d7bdf4f62\\\\x2dc77c\\\\x2dfa88\\\\x2d985c\\\\x2d7b9d5312da7f.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689546124356
Jul 17 03:23:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:19.406Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"update.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689546124356
Jul 17 03:23:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:19.715Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"tsg_master_T_drop_log\", app_name=\"tsg_master\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"traffic-engine\", line_name=\"BGP-RECORD\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", table_name=\"sendlog\", vsysId=\"1\"}" timestamp=1689546133544
Jul 17 03:23:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:19.720Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_cache\", asset=\"twa-tsgx06\", asset_id=\"101\", container=\"dign-server-dns\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546114508
Jul 17 03:23:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:19.735Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_bytes_total\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"cadvisor\", interface=\"ens8f0np0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546129159
Jul 17 03:23:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:19.756Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"113\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546137612
Jul 17 03:23:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:19.768Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"86\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546137612
Jul 17 03:23:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:19.799Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-2199681b\\\\x2d3e95\\\\x2d4349\\\\x2d8d1f\\\\x2d0c2eb8c38c0f-volumes-kubernetes.io\\\\x7esecret-config.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689546124356
Jul 17 03:23:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:19.920Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"shm\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/e2a96b8504cd363c5eb4f87c7995b0a0f4c283f07fc4af8d79131040b03ce6a7/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546137612
Jul 17 03:23:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:19.921Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"11\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546137612
Jul 17 03:23:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:19.933Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"3\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546137612
Jul 17 03:23:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:19.953Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link_mode\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth3177b6ed\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546137612
Jul 17 03:23:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:19.967Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-infiniband-umad6.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689546124356
Jul 17 03:23:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:20.023Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.grpc.v1.cri-sandboxes-5dbf404b489550de3a53fed4a38759fd123b29cdda7e1bfacf96894b3dba4909-shm.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689546124356
Jul 17 03:23:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:20.035Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:20-0000:20:03.1-0000:21:00.1-net-ens2f1np1.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689546124356
Jul 17 03:23:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:20.071Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-4ce0222b\\\\x2d34a2\\\\x2d45cf\\\\x2dbbe5\\\\x2dfd12c5e4d780-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2d69txz.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689546124356
Jul 17 03:23:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:20.237Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"46\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546137612
Jul 17 03:23:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:20.318Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_annotations\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_endpoint=\"shaping-announce-port-tsg-traffic-engine-numa-1\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546133949
Jul 17 03:23:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:20.554Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"86\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546128263
Jul 17 03:23:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:20.718Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_spec_replicas\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"prometheus-alertmanager\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546133949
Jul 17 03:23:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:20.744Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"127\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546137612
Jul 17 03:23:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:20.852Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"113\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546137612
Jul 17 03:23:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:20.915Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"51\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546128263
Jul 17 03:23:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:21.070Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546128263
Jul 17 03:23:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:21.166Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_bytes_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"vethbc6c7302\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546128263
Jul 17 03:23:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:21.197Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"29\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546128263
Jul 17 03:23:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:21.328Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-e2a96b8504cd363c5eb4f87c7995b0a0f4c283f07fc4af8d79131040b03ce6a7-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689546124356
Jul 17 03:23:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:21.382Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-ttyS2.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689546130657
Jul 17 03:23:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:21.590Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", cpu=\"47\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"OLAP-Node-Exporter-twa-dellr740-dt001\", endpoint_id=\"257\", mode=\"nice\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546141405
Jul 17 03:23:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:21.789Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", container=\"ipmi-exporter\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"cadvisor\", failure_type=\"pgmajfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"container\"}" timestamp=1689546120177
Jul 17 03:23:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:21.805Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_limit_bytes\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda5\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546131539
Jul 17 03:23:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:21.824Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"pcap-tsgx10\", asset_id=\"89\", condition=\"Progressing\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"coredns\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"false\"}" timestamp=1689546133828
Jul 17 03:23:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:21.832Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_reads_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", container=\"metrics-server\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546124142
Jul 17 03:23:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:21.926Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-node-exporter-kd42j\", project=\"TSG-OS\", project_id=\"7\", reason=\"NodeLost\", service=\"prometheus-kube-state-metrics\", uid=\"3babe650-f538-4383-8bda-a77b2daf72dc\"}" timestamp=1689546133828
Jul 17 03:23:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:22.025Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_oom_events_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", container=\"monit-stream\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546128423
Jul 17 03:23:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:22.037Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"smartd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"service\"}" timestamp=1689546130657
Jul 17 03:23:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:22.063Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-80be009ae9aab0270cea9b98d94e0627ae6f5e2d2d66570b5bba92ebaf1ceed2.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"scope\"}" timestamp=1689546130657
Jul 17 03:23:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:22.545Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_goroutines\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"OLAP-Node-Exporter-twa-dellr740-dt001\", endpoint_id=\"257\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546141405
Jul 17 03:23:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:22.975Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_info\", address=\"00:00:00:00:00:00\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", broadcast=\"00:00:00:00:00:00\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"lo\", endpoint=\"OLAP-Node-Exporter-twa-dellr740-dt001\", endpoint_id=\"257\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", operstate=\"unknown\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546141405
Jul 17 03:23:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:23.486Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_hwmon_sensor_label\", asset=\"msh-tsgx09\", asset_id=\"55\", chip=\"pci0000:00_0000:00:19_3\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", label=\"tccd8\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", sensor=\"temp10\", service=\"prometheus-node-exporter\"}" timestamp=1689546134790
Jul 17 03:23:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:23.507Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_cluster_enabled\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546137019
Jul 17 03:23:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:23.532Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_commands_duration_seconds_total\", asset=\"msh-kvm-cm001\", asset_id=\"3\", cmd=\"zscore\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"CM-Sync-Redis-msh-kvm-cm001\", endpoint_id=\"307\", host=\"192.168.20.157\", module=\"CM-Sync-Redis\", module_id=\"67\", nz_agent_id=\"3\", parent_asset=\"msh-dellr740-pve001\", parent_asset_id=\"1\", project=\"Central-Management\", project_id=\"5\"}" timestamp=1689546143344
Jul 17 03:23:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:23.568Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"NetworkManager-wait-online.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"service\"}" timestamp=1689546138416
Jul 17 03:23:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:23.685Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_errs_total\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"vethd0f6e715\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546134790
Jul 17 03:23:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:23.749Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db16\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546137019
Jul 17 03:23:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:24.081Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_info\", asset=\"msh-tsgx09\", asset_id=\"55\", configmap=\"telegraf-tsg-traffic-engine-numa-1\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546140068
Jul 17 03:23:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:24.092Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_address_available\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_endpoint=\"prometheus-node-exporter\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546140068
Jul 17 03:23:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:24.137Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_free_bytes\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"shm\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/c40b2309c119752ef676fcbe43a14be426835fa3b87e6462f17f440fbf7c8014/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546134790
Jul 17 03:23:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:24.246Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_fifo_total\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"virtio_dign_s\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546134790
Jul 17 03:23:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:24.371Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx09\", asset_id=\"55\", cpu=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546134790
Jul 17 03:23:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:24.451Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"msh-tsgx09\", asset_id=\"55\", cpu=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546134790
Jul 17 03:23:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:24.457Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_readonly\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"shm\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/fb0b02b65b71a7043764d1659b7950236f88ec1c01336a745a9f38af6148685a/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546134790
Jul 17 03:23:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:24.480Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_annotations\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"kubernetes-dashboard\", uid=\"74413472-e3ff-44d4-b9a6-c1c4ec5b112d\"}" timestamp=1689546140068
Jul 17 03:23:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:24.502Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"msh-tsgx09\", asset_id=\"55\", container=\"telegraf\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", state=\"running\"}" timestamp=1689546131079
Jul 17 03:23:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:24.531Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"2\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546134790
Jul 17 03:23:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:24.534Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"shm\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/83286f721fcb3123fe7ecffc53f05572b024bd12ed7f95ba49a4871f75d356cb/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546134790
Jul 17 03:23:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:24.580Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-netns-cni\\\\x2dc0061069\\\\x2d558d\\\\x2da594\\\\x2df89b\\\\x2daa2c7f271103.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689546138416
Jul 17 03:23:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:24.719Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"msh-tsgx09\", asset_id=\"55\", condition=\"false\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-systemd-exporter-2h74t\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"75668a8f-85fc-45c3-9b22-ebd6bba21c17\"}" timestamp=1689546140068
Jul 17 03:23:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:24.735Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:80-0000:80:01.2-0000:82:00.2-net-ens4f2np2.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689546138416
Jul 17 03:23:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:24.850Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_hwmon_chip_names\", asset=\"msh-tsgx09\", asset_id=\"55\", chip=\"0000:60:03_1_0000:63:00_0\", chip_name=\"bnxt_en\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546134790
Jul 17 03:23:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:24.865Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"msh-tsgx09\", asset_id=\"55\", cpu=\"107\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546134790
Jul 17 03:23:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:24.883Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2did-wwn\\\\x2d0x62cea7f0848263002c2c205714e9cdc4\\\\x2dpart3.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689546138416
Jul 17 03:23:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:24.905Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"tmpfs\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/data/var/lib/kubelet/pods/ab6b1591-cfaa-4964-8fcc-a861d0862792/volumes/kubernetes.io~projected/kube-api-access-p2tkk\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546140160
Jul 17 03:23:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:24.914Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"109\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546134790
Jul 17 03:23:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:24.943Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"abrt-journal-core.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689546138416
Jul 17 03:23:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:24.980Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"msh-tsgx09\", asset_id=\"55\", cpu=\"48\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546134790
Jul 17 03:23:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:25.003Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2dpath-pci\\\\x2d0000:81:00.0\\\\x2dscsi\\\\x2d0:2:0:0\\\\x2dpart4.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689546138416
Jul 17 03:23:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:25.057Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_speed_bytes\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth0c6b6d72\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546134790
Jul 17 03:23:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:25.112Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", state=\"sleeping\"}" timestamp=1689546124163
Jul 17 03:23:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:25.233Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_packets_total\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", interface=\"ens6f0np0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546137254
Jul 17 03:23:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:25.313Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"0\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546140160
Jul 17 03:23:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:25.359Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_timex_sync_status\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546140160
Jul 17 03:23:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:25.379Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_labels\", asset=\"pcap-tsgx12\", asset_id=\"117\", configmap=\"kubernetes-dashboard-settings\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546131122
Jul 17 03:23:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:25.452Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"auditd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"service\"}" timestamp=1689546131102
Jul 17 03:23:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:25.497Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"configfs\", name=\"sys-kernel-config.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689546138416
Jul 17 03:23:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:25.506Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"89\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546134790
Jul 17 03:23:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:25.762Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"nfs-utils.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"oneshot\", type=\"service\"}" timestamp=1689546131102
Jul 17 03:23:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:25.987Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"56\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546135810
Jul 17 03:23:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:26.230Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"jvm_memory_bytes_used\", area=\"heap\", asset=\"pcap-dellr740-dt003\", asset_id=\"35\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-Yarn-pcap-dellr740-dt003-NodeManager\", endpoint_id=\"361\", module=\"DC-Yarn\", module_id=\"1\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546145525
Jul 17 03:23:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:26.269Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_annotations\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-1-firewall-846ccb5cd9\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546138744
Jul 17 03:23:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:26.324Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"47\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546135810
Jul 17 03:23:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:26.621Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"phy_rx_error_total\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", dev=\"ens1f1np1\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"mrzcpd-exporter\"}" timestamp=1689546138295
Jul 17 03:23:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:26.657Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_exporter_scrape_duration_seconds_sum\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546143549
Jul 17 03:23:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:26.670Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2dpartlabel-TSG\\\\x2dOS\\\\x2dSYSROOT.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689546133907
Jul 17 03:23:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:26.758Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_oom_events_total\", asset=\"twa-tsgx09\", asset_id=\"107\", container=\"monit-stream\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546129642
Jul 17 03:23:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:26.783Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"7\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546135810
Jul 17 03:23:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:26.815Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"63\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546135810
Jul 17 03:23:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:26.834Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"86\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546135810
Jul 17 03:23:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:27.032Z caller=dedupe.go:112 component=remote level=error remote_name=52db6f url=http://10.10.10.159:10090/nz-agent/prometheus/proxy/api/v1/write msg="non-recoverable error" count=1000 exemplarCount=0 err="server returned HTTP status 400 : out of order sample"
Jul 17 03:23:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:27.121Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"80\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546135810
Jul 17 03:23:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:27.186Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_restart_policy\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"metrics-server-54dc485875-24t2d\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", type=\"Always\", uid=\"25066531-7d86-4be0-be75-7e9052465c16\"}" timestamp=1689546138744
Jul 17 03:23:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:27.189Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_queue_length\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens3f1np1\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546135810
Jul 17 03:23:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:27.225Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_state_id\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_7\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546135810
Jul 17 03:23:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:27.251Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dracut-pre-udev.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"oneshot\", type=\"service\"}" timestamp=1689546133907
Jul 17 03:23:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:27.489Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"msh-tsgx10\", asset_id=\"69\", condition=\"Available\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"mrzcpd-exporter\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"unknown\"}" timestamp=1689546135682
Jul 17 03:23:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:27.593Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_ready_replicas\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"cm-policy-local-cache-redis-master-69967b5879\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546135682
Jul 17 03:23:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:27.696Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"dashboard-metrics-scraper\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Sync\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546138346
Jul 17 03:23:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:27.857Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"firewall\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Total\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689546144217
Jul 17 03:23:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:27.894Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_created\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-pushgateway-bd476bcf9-jkpjd\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"7ac64d36-f972-409d-8e70-7eed1da8b973\"}" timestamp=1689546135682
Jul 17 03:23:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:27.980Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_metadata_resource_version\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"k3s-serving\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546135682
Jul 17 03:23:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:28.352Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_gc_duration_seconds\", agent_name=\"msh-kvm-nz001\", application=\"loki\", datacenter=\"MSH\", instance=\"127.0.0.1:13100\", quantile=\"0.25\"}" timestamp=1689546148331
Jul 17 03:23:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:28.726Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_system_seconds_total\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"dign-client\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546140990
Jul 17 03:23:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:28.732Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_created\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"mrzcpd-exporter-5cf7f967cc\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546135682
Jul 17 03:23:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:28.746Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db45\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546139446
Jul 17 03:23:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:28.759Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_speed_bytes\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth30e28f20\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546142882
Jul 17 03:23:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:28.992Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_metadata_generation\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"mrzcpd-exporter-5cf7f967cc\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546135682
Jul 17 03:23:29 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:29.045Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_writes_bytes_total\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"traefik\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546129818
Jul 17 03:23:30 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:30.920Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kafka_server_BrokerTopicMetrics_OneMinuteRate\", asset=\"pcap-dellr740-dt003\", asset_id=\"35\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-Kafka-pcap-dellr740-dt003\", endpoint_id=\"103\", module=\"DC-Kafka\", module_id=\"5\", name=\"FetchMessageConversionsPerSec\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", topic=\"ACTIVE-DEFENCE-EVENT\"}" timestamp=1689546149946
Jul 17 03:23:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:31.003Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db73\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546150141
Jul 17 03:23:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:31.059Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_load1\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546140939
Jul 17 03:23:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:31.141Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_multicast_total\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"vethef1c04ca\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546140939
Jul 17 03:23:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:31.177Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-29d761a8b45c89a5c0dbbbd6417587a5e5d3521c3aadeb9ca997c93043a091df.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"scope\"}" timestamp=1689546141015
Jul 17 03:23:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:31.301Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_metadata_generation\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"prometheus-alertmanager\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546144068
Jul 17 03:23:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:31.314Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_info\", asset=\"msh-tsgx05\", asset_id=\"67\", container=\"dign-server-dns\", container_id=\"containerd://5d02fdd60d146575a4d80f643a90574e02f67e254771de3d8ac160899055b9d9\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", image=\"docker.io/library/dign-server-dns:latest\", image_id=\"sha256:6987ac49c2d264e9289120b41e4a48536092ce6fbe41c981cdd4ad115feae8a4\", image_spec=\"dign-server-dns:latest\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"dign-server-t24gc\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"1d72bbea-870a-432c-b71c-26fca67b7d6c\"}" timestamp=1689546144068
Jul 17 03:23:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:31.463Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_info\", asset=\"twa-tsgx03\", asset_id=\"95\", configmap=\"chart-content-cm-policy-local-cache\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546150436
Jul 17 03:23:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:31.481Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_connections_received_total\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546146438
Jul 17 03:23:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:31.575Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled_time\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"configmap-reloader-6c55bbc49f-cmv7r\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"6d138495-2ef4-43b8-aa40-820b8611ffff\"}" timestamp=1689546144068
Jul 17 03:23:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:31.592Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_ready\", asset=\"msh-tsgx05\", asset_id=\"67\", container=\"prometheus-server\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-server-6ff784d758-rmvjl\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"0fa5c602-236e-4657-a253-bfddabdcc9f1\"}" timestamp=1689546144068
Jul 17 03:23:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:31.611Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-veth283b535f.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689546141015
Jul 17 03:23:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:31.639Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"86\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546140939
Jul 17 03:23:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:31.654Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"26\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546139093
Jul 17 03:23:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:31.668Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_packets_total\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens6f0np0\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546139093
Jul 17 03:23:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:31.746Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_spec_replicas\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"metrics-server\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546150436
Jul 17 03:23:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:31.919Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_id\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth4eba9fba\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546140939
Jul 17 03:23:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:31.973Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kafka_network_RequestMetrics_Errors_total\", asset=\"pcap-dellr740-dt003\", asset_id=\"35\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-Kafka-pcap-dellr740-dt003\", endpoint_id=\"103\", error=\"REBALANCE_IN_PROGRESS\", module=\"DC-Kafka\", module_id=\"5\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", request=\"Heartbeat\"}" timestamp=1689546149946
Jul 17 03:23:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:31.985Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_init_container_status_ready\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"install-multus-binary\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"kube-multus-ds-hhvw7\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"3c517891-c7a5-4f99-9730-42d4a10fa392\"}" timestamp=1689546139276
Jul 17 03:23:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:31.997Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_last_seen\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"metrics\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546142033
Jul 17 03:23:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:32.017Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_swap\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"traefik\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546130717
Jul 17 03:23:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:32.029Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"msh-tsgx05\", asset_id=\"67\", container=\"prometheus-node-exporter\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"stopped\"}" timestamp=1689546130355
Jul 17 03:23:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:32.086Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_oom_events_total\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689546140734
Jul 17 03:23:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:32.086Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"msh-tsgx11\", asset_id=\"111\", condition=\"unknown\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-kube-state-metrics-6b768c7bf8-mjhbh\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"c9b0dc64-ed75-445b-a713-ada2b43ce324\"}" timestamp=1689546139276
Jul 17 03:23:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:32.104Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failcnt\", asset=\"twa-tsgx03\", asset_id=\"95\", container=\"prometheus-pushgateway\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546136065
Jul 17 03:23:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:32.109Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"nfs-utils.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"service\"}" timestamp=1689546142644
Jul 17 03:23:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:32.129Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_labels\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"local-path-provisioner-957fdf8bc\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546139276
Jul 17 03:23:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:32.178Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_system_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", container=\"lb-tcp-443\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546140073
Jul 17 03:23:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:32.182Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_errors_total\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"cadvisor\", interface=\"ens4f2np2\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546139213
Jul 17 03:23:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:32.194Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"106\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546140939
Jul 17 03:23:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:32.209Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_flags\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"vethc8e6a12a\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546140939
Jul 17 03:23:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:32.229Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kdump.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"service\"}" timestamp=1689546146907
Jul 17 03:23:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:32.335Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_mspan_sys_bytes\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"ipmi-exporter\"}" timestamp=1689546139925
Jul 17 03:23:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:32.349Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"shm\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/e36b58fd14a8bd05ba984cc788feefbfbdc39342d0281d2d99647137cb7ce132/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546140939
Jul 17 03:23:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:32.371Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_memory_Inactive_bytes\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546140939
Jul 17 03:23:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:32.373Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_complete\", asset=\"twa-tsgx03\", asset_id=\"95\", condition=\"true\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", job_name=\"helm-install-mrzcpd-exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546150436
Jul 17 03:23:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:32.387Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_virtual_memory_bytes\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"snmpd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689546141015
Jul 17 03:23:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:32.441Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db244\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546146438
Jul 17 03:23:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:32.457Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"overlay\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-823a9850cf952fec07f6e1b9e107f7e86acacfac1e6dcbb4539307bed767287f-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689546146907
Jul 17 03:23:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:32.457Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_cpu_seconds_total\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"slice\"}" timestamp=1689546141015
Jul 17 03:23:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:32.475Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_owner\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", owner_is_controller=\"true\", owner_kind=\"Deployment\", owner_name=\"tsg-traffic-engine-numa-0-firewall\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-0-firewall-686c8b6fbd\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546150436
Jul 17 03:23:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:32.618Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-module-configfs.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689546142644
Jul 17 03:23:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:32.635Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"42\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546139093
Jul 17 03:23:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:32.657Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_cpu_period\", asset=\"twa-tsgx05\", asset_id=\"99\", container=\"prometheus-server-configmap-reload\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546138535
Jul 17 03:23:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:32.662Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_compressed_total\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"cnibr0\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546140791
Jul 17 03:23:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:32.678Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"80\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546140605
Jul 17 03:23:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:32.682Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db68\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546136711
Jul 17 03:23:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:32.683Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_ports\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_endpoint=\"cm-policy-local-cache-redis-master\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", port_name=\"tcp-redis\", port_number=\"6379\", port_protocol=\"TCP\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546150436
Jul 17 03:23:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:32.702Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_cpu_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-pode02c929a_3c07_4477_b04a_8ffc16c405f5.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"slice\"}" timestamp=1689546147691
Jul 17 03:23:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:32.721Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"plymouth-switch-root.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689546147691
Jul 17 03:23:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:32.750Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"56\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546140605
Jul 17 03:23:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:32.750Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.grpc.v1.cri-sandboxes-b5752670953447bd514584f04f4322c5c31b239d7238666801032ee1a2ac736e-shm.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689546147691
Jul 17 03:23:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:32.754Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_errs_total\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens1f1np1\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546140605
Jul 17 03:23:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:32.778Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-pod37526140_01d6_46f5_9f0b_e066a6254bac.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"slice\"}" timestamp=1689546147691
Jul 17 03:23:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:32.785Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_changes_total\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"vethe4b4946d\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546140791
Jul 17 03:23:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:32.791Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_readonly\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"shm\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/5591d299f69c80e8d5c4ca725033b77a1b7841e75eed16565ee8de76e814b6aa/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546140605
Jul 17 03:23:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:32.814Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.grpc.v1.cri-sandboxes-1242875f9ccc3d5ab62db93f916bd67c06da5b8e47f89d7f923c00b4c43c725a-shm.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689546147691
Jul 17 03:23:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:32.847Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"38\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546140605
Jul 17 03:23:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:32.859Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_errors_total\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", interface=\"ens4f1np1\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546140682
Jul 17 03:23:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:32.862Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_last_seen\", asset=\"twa-tsgx03\", asset_id=\"95\", container=\"telegraf\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689546143143
Jul 17 03:23:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:32.862Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-vconsole-setup.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"service\"}" timestamp=1689546142644
Jul 17 03:23:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:32.957Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_virtual_memory_bytes\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"packagekit.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689546142644
Jul 17 03:23:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:33.066Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_owner\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", job_name=\"helm-install-prometheus\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", owner_is_controller=\"<none>\", owner_kind=\"<none>\", owner_name=\"<none>\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546139276
Jul 17 03:23:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:33.088Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-veth5dd438f9.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689546147691
Jul 17 03:23:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:33.100Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_mtu_bytes\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethb98a16e3\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546140605
Jul 17 03:23:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:33.107Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"snmptrapd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689546142644
Jul 17 03:23:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:33.157Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db164\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546149260
Jul 17 03:23:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:33.202Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"udisks2.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"service\"}" timestamp=1689546147691
Jul 17 03:23:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:33.261Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"nss-user-lookup.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"target\"}" timestamp=1689546147691
Jul 17 03:23:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:33.279Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_sockets\", asset=\"twa-tsgx05\", asset_id=\"99\", container=\"metrics-server\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546127980
Jul 17 03:23:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:33.292Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"twa-tsgx05\", asset_id=\"99\", container=\"telegraf-security\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", state=\"iowaiting\"}" timestamp=1689546131649
Jul 17 03:23:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:33.804Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-bdd10b0d\\\\x2dcc33\\\\x2d4a4f\\\\x2d9622\\\\x2d89ebf267e6c3-volume\\\\x2dsubpaths-config\\\\x2dvolume-telegraf-2.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689546147691
Jul 17 03:23:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:33.849Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-netns-cni\\\\x2d8176b129\\\\x2dba87\\\\x2d4783\\\\x2d0ff8\\\\x2dcfdff9772adc.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689546147691
Jul 17 03:23:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:34.186Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2dlabel-ONIE\\\\x2dBOOT.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689546147691
Jul 17 03:23:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:34.214Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-cnibr0.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689546147691
Jul 17 03:23:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:34.327Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"tmpfs\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/data/var/lib/kubelet/pods/4984267b-c91e-4d03-b7d6-3defa3eedc21/volumes/kubernetes.io~projected/kube-api-access-mmq6w\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546140605
Jul 17 03:23:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:34.360Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_unicast_packets_transmitted_total\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_4\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546140605
Jul 17 03:23:39 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:39.497Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-kvm-cm001\", asset_id=\"3\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db109\", endpoint=\"CM-Branch-Redis-msh-kvm-cm001\", endpoint_id=\"319\", host=\"192.168.20.157\", module=\"CM-Branch-Redis\", module_id=\"73\", nz_agent_id=\"3\", parent_asset=\"msh-dellr740-pve001\", parent_asset_id=\"1\", project=\"Central-Management\", project_id=\"5\"}" timestamp=1689546158579
Jul 17 03:23:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:40.493Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_metadata_resource_version\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.tsg-traffic-engine-numa-1.v3\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546160034
Jul 17 03:23:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:40.548Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_errors_total\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"cadvisor\", interface=\"bond0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546146499
Jul 17 03:23:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:40.565Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_cpu_period\", asset=\"twa-tsgx04\", asset_id=\"97\", container=\"redis\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546148797
Jul 17 03:23:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:40.588Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-alertmanager-6df7f457b6-vphf9\", project=\"TSG-OS\", project_id=\"7\", reason=\"UnexpectedAdmissionError\", service=\"prometheus-kube-state-metrics\", uid=\"94a4f272-a146-4944-832c-92f8c3e47728\"}" timestamp=1689546160034
Jul 17 03:23:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:40.624Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546151647
Jul 17 03:23:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:40.852Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_arp_entries\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"cnibr0\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546151647
Jul 17 03:23:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:41.018Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Dup_Udp_Pkt\", app_name=\"sapp4\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", vsysId=\"1\"}" timestamp=1689546159161
Jul 17 03:23:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:41.190Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_FlushMemstoreSize_90th_percentile\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Server\"}" timestamp=1689546159756
Jul 17 03:23:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:41.418Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"atd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"service\"}" timestamp=1689546155764
Jul 17 03:23:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:41.438Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-fsck-root.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"service\"}" timestamp=1689546155764
Jul 17 03:23:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:41.489Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_FsReadTime_75th_percentile\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"IO\"}" timestamp=1689546159756
Jul 17 03:23:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:41.753Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"twa-tsgx04\", asset_id=\"97\", condition=\"Progressing\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"prometheus-server\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"unknown\"}" timestamp=1689546160034
Jul 17 03:23:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:41.778Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_maxFlushQueueSize\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", namespace=\"SYSTEM\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"c615391350344bb3d21bd402ab577935\", table=\"LOG\", type=\"data\"}" timestamp=1689546159756
Jul 17 03:23:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:41.802Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_hbase_table_meta_metric_compactionTime_min\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546159756
Jul 17 03:23:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:41.864Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_10_194_1_metric_putBatchTime_90th_percentile\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546159756
Jul 17 03:23:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:41.884Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_10_194_1_metric_splitTime_99th_percentile\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546159756
Jul 17 03:23:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:41.928Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_PutBatch_98th_percentile\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Server\"}" timestamp=1689546159756
Jul 17 03:23:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:41.996Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_cpu_shares\", asset=\"twa-tsgx04\", asset_id=\"97\", container=\"coredns\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546148797
Jul 17 03:23:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:42.018Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"twa-tsgx04\", asset_id=\"97\", condition=\"Progressing\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"prometheus-kube-state-metrics\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"true\"}" timestamp=1689546160034
Jul 17 03:23:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:42.054Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_running\", asset=\"twa-tsgx04\", asset_id=\"97\", container=\"helm\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-cm-policy-local-cache-rxfmw\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"9e8ca26d-6038-4672-99ec-32891ca961a5\"}" timestamp=1689546160034
Jul 17 03:23:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:42.098Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_threads_max\", asset=\"twa-tsgx04\", asset_id=\"97\", container=\"prometheus-node-exporter\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546148791
Jul 17 03:23:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:42.124Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db190\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546153827
Jul 17 03:23:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:42.573Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_info\", asset=\"pcap-tsgx01\", asset_id=\"71\", created_by_kind=\"ReplicaSet\", created_by_name=\"coredns-79c6ccb74b\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", host_ip=\"10.10.10.161\", host_network=\"false\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", node=\"localhost\", nz_agent_id=\"3\", pod=\"coredns-79c6ccb74b-rpzl7\", pod_ip=\"10.42.0.54\", priority_class=\"system-cluster-critical\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"be245aed-e373-405f-8d79-93c5764d578b\"}" timestamp=1689546155154
Jul 17 03:23:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:42.590Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"pcap-tsgx01\", asset_id=\"71\", condition=\"false\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"systemd-exporter-7f549d9485-7kvgn\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"27368c72-efff-405c-9fe0-1a20af809631\"}" timestamp=1689546155154
Jul 17 03:23:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:42.627Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"rescue.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"idle\", type=\"service\"}" timestamp=1689546160671
Jul 17 03:23:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:42.768Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_commands_rejected_calls_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", cmd=\"exec\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546154838
Jul 17 03:23:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:42.947Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_writes_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda5\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546155751
Jul 17 03:23:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:43.503Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_address_available\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_endpoint=\"dashboard-metrics-scraper\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546155154
Jul 17 03:23:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:43.534Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_init_container_status_running\", asset=\"pcap-tsgx01\", asset_id=\"71\", container=\"tsg-dign-init\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"dign-server-xv84k\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"d57d31b4-5150-4e58-8d32-5e87c6e75897\"}" timestamp=1689546155154
Jul 17 03:23:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:43.679Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", cpu=\"80\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546156242
Jul 17 03:23:44 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:44.076Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_time_seconds\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546156242
Jul 17 03:23:44 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:44.093Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-4aa609e68f99f63f39577529dd0ad4b8e01d33713af324dd9a3161f0ac85fc5d.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"scope\"}" timestamp=1689546160671
Jul 17 03:23:44 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:44.114Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", cpu=\"3\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Gohangout-msh-dellr740-dt002\", endpoint_id=\"167\", mode=\"steal\", module=\"DC-Gohangout\", module_id=\"35\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546162623
Jul 17 03:23:44 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:44.166Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", container=\"coredns\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Discard\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546146746
Jul 17 03:23:44 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:44.184Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Succeeded\", pod=\"helm-install-cm-policy-local-cache-7kss5\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"4d79d568-e597-4292-9848-bc8f41715e7a\"}" timestamp=1689546155154
Jul 17 03:23:46 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:46.742Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"jvm_buffer_pool_used_bytes\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-Yarn-pcap-dellr740-dt001-ResourceManager\", endpoint_id=\"135\", module=\"DC-Yarn\", module_id=\"1\", nz_agent_id=\"3\", pool=\"mapped\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546165217
Jul 17 03:23:48 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:48.694Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_193_1_metric_tableWriteQueryPerSecond_15min_rate\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546167440
Jul 17 03:23:48 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:48.747Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_tsg_galaxy_table_gtpc_knowledge_base_metric_scanSize_median\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546167440
Jul 17 03:23:48 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:48.760Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_memStoreSize\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"knowledge_base_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"071445389ef0f0e9dc0ea8dab1b4d0f6\", table=\"index_time_knowledge_base_hos_bucket\", type=\"index\"}" timestamp=1689546167440
Jul 17 03:23:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:49.160Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_CheckAndDelete_95th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Server\"}" timestamp=1689546167440
Jul 17 03:23:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:49.450Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_deleteCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", namespace=\"SYSTEM\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"d330648f7279c8a62731bd0e25e320c5\", table=\"LOG\", type=\"data\"}" timestamp=1689546167440
Jul 17 03:23:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:49.494Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_avgStoreFileAge\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"firewall_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"64070c81c737589d75cf08334714566f\", table=\"index_time_firewall_hos_bucket\", type=\"index\"}" timestamp=1689546167440
Jul 17 03:23:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:49.604Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_flushesQueuedCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"b7c2fa322dfc9707103829994af6fdc1\", table=\"rtp_hos_bucket\", type=\"data\"}" timestamp=1689546167440
Jul 17 03:23:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:49.663Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_oom_events_total\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"kube-multus\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546136728
Jul 17 03:23:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:49.795Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas_available\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"kubernetes-dashboard\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546159668
Jul 17 03:23:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:49.817Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_reads_total\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda2\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546151801
Jul 17 03:23:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:49.830Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_replicas\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"coredns-766944d45b\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546159668
Jul 17 03:23:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:49.942Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_numFilesCompactedCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"default\", project=\"OLAP\", project_id=\"3\", region=\"d83cc1aea06dd989e81e7b9e4481ca14\", table=\"index_filename_default\", type=\"index\"}" timestamp=1689546167440
Jul 17 03:23:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:49.960Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_filteredReadRequestCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"2e3b5c758c7204f809dda8782c42c59e\", table=\"proxy_hos_bucket\", type=\"data\"}" timestamp=1689546167440
Jul 17 03:23:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:49.974Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_session_record_hos_bucket_metric_majorCompactionInputFileCount_max\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546167440
Jul 17 03:23:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:49.984Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_UnblockedFlushes_25th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Memory\"}" timestamp=1689546167440
Jul 17 03:23:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:50.001Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_compactionsFailedCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"proxy_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"a432dccdd145b57edf2f616d7fb38ab3\", table=\"index_filename_proxy_hos_bucket\", type=\"index\"}" timestamp=1689546167440
Jul 17 03:23:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:50.037Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_193_1_metric_incrementTime_90th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546167440
Jul 17 03:23:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:50.104Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_hbase_table_namespace_metric_majorCompactionInputSize_75th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546167440
Jul 17 03:23:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:50.129Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_swap\", asset=\"twa-tsgx08\", asset_id=\"105\", container=\"kube-multus\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546162812
Jul 17 03:23:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:50.169Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_getCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", namespace=\"SYSTEM\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"d330648f7279c8a62731bd0e25e320c5\", table=\"LOG\", type=\"data\"}" timestamp=1689546167440
Jul 17 03:23:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:50.201Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_2_metric_majorCompactionOutputSize_num_ops\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546167440
Jul 17 03:23:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:50.230Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"75\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546166709
Jul 17 03:23:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:50.337Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_scanCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"assessment_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"5c26fb1184599cdf5d7614ebef291569\", table=\"index_partfile_assessment_hos_bucket\", type=\"index\"}" timestamp=1689546167440
Jul 17 03:23:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:50.408Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_default_metric_tableWriteQueryPerSecond_1min_rate\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546167440
Jul 17 03:23:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:50.441Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_193_1_metric_putTime_75th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546167440
Jul 17 03:23:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:50.462Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_vl15_dropped_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_8\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546166709
Jul 17 03:23:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:50.488Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_state_started\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"dign-client\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"dign-client-qd7p8\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"1b9e7e11-9912-45e0-831b-8143d0877131\"}" timestamp=1689546159668
Jul 17 03:23:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:50.508Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_disk_flush_requests_time_seconds_total\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"sda\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546161797
Jul 17 03:23:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:50.527Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"twa-tsgx10\", asset_id=\"109\", cpu=\"117\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546161797
Jul 17 03:23:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:50.527Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx10\", asset_id=\"109\", cpu=\"24\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546161797
Jul 17 03:23:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:50.546Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_scrape_collector_duration_seconds\", asset=\"twa-tsgx08\", asset_id=\"105\", collector=\"tapestats\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546166709
Jul 17 03:23:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:50.550Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_ips\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", ip=\"10.42.0.119\", ip_family=\"4\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-tsg-traffic-engine-numa-1-7p4wv\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"402360b4-e436-4486-bb15-dd73b4647755\"}" timestamp=1689546161568
Jul 17 03:23:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:50.553Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx10\", asset_id=\"109\", cpu=\"116\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546161797
Jul 17 03:23:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:50.706Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"71\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546166709
Jul 17 03:23:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:50.775Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_io_current\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/var/lib/kubelet/pods/c4ec4f52-224c-4f5b-9432-db80b6600279/volumes/kubernetes.io~projected/kube-api-access-g4sbf\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546160113
Jul 17 03:23:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:50.795Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_working_set_bytes\", asset=\"twa-tsgx08\", asset_id=\"105\", container=\"ipmi-exporter\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546160789
Jul 17 03:23:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:50.845Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_files\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda3\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", fstype=\"ext4\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/update\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546166709
Jul 17 03:23:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:50.860Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_device_id\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"eno1\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546166709
Jul 17 03:23:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:50.874Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_down_changes_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"br_dign_c\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546166709
Jul 17 03:23:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:50.883Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_threads_max\", asset=\"twa-tsgx08\", asset_id=\"105\", container=\"lb-tcp-443\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546163466
Jul 17 03:23:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:50.886Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_nf_conntrack_stat_insert\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-olap003\", endpoint_id=\"235\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"yarn_resourcemanager,zookeeper,nacos,kafka,spark,druid_data,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546169713
Jul 17 03:23:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:50.918Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_max_fds\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"exporter-systemd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689546166554
Jul 17 03:23:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:50.951Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_mtu_bytes\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens1f1np1\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546166709
Jul 17 03:23:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:51.099Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_running\", asset=\"twa-tsgx08\", asset_id=\"105\", container=\"dign-client\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"dign-client-pv7jg\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"f9e681d8-8372-4551-8580-4a68b3fae8df\"}" timestamp=1689546161568
Jul 17 03:23:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:51.102Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_constraint_errors_transmitted_total\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_7\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546161797
Jul 17 03:23:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:51.145Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_data_transmitted_bytes_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_11\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546166709
Jul 17 03:23:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:51.166Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"twa-tsgx08\", asset_id=\"105\", condition=\"Available\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"systemd-exporter\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"unknown\"}" timestamp=1689546161568
Jul 17 03:23:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:51.185Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_carrier_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens1f1np1\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546166709
Jul 17 03:23:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:51.189Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_restart_policy\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"dign-client-pv7jg\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", type=\"Always\", uid=\"f9e681d8-8372-4551-8580-4a68b3fae8df\"}" timestamp=1689546161568
Jul 17 03:23:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:51.220Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_queue_length\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth83674fd8\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546161797
Jul 17 03:23:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:51.229Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_annotations\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"chart-values-tsg-traffic-engine-numa-0\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546159668
Jul 17 03:23:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:51.313Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-podee4ec55a_f496_4629_b299_cf918d7658cb.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"slice\"}" timestamp=1689546166554
Jul 17 03:23:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:51.346Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"tmpfs\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/var/lib/kubelet/pods/69257332-0d52-4d37-98ae-1ff0fd197fb1/volumes/kubernetes.io~projected/kube-api-access-v48ld\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546161797
Jul 17 03:23:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:51.351Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"twa-tsgx08\", asset_id=\"105\", container=\"exporter-merger\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", failure_type=\"pgmajfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"container\"}" timestamp=1689546158743
Jul 17 03:23:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:51.383Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_labels\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"prometheus-pushgateway\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546161568
Jul 17 03:23:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:51.399Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db63\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546167556
Jul 17 03:23:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:51.465Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"twa-tsgx08\", asset_id=\"105\", condition=\"Progressing\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"prometheus-server\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"unknown\"}" timestamp=1689546161568
Jul 17 03:23:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:51.498Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2dpartlabel-TSG\\\\x2dOS\\\\x2dSYSROOT.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689546166554
Jul 17 03:23:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:51.569Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.grpc.v1.cri-sandboxes-0500205352582fe55f0c9607f3e3848e1f23c3fd83667094351e6d099fc71fcc-shm.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689546166554
Jul 17 03:23:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:51.580Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-tmpfiles-setup-dev.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"service\"}" timestamp=1689546166554
Jul 17 03:23:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:52.114Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_minStoreFileAge\", asset=\"pcap-dellr740-dt002\", asset_id=\"33\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt002-HregionServer\", endpoint_id=\"155\", module=\"DC-HBase\", module_id=\"3\", namespace=\"SYSTEM\", nz_agent_id=\"1\", project=\"OLAP\", project_id=\"3\", region=\"1e80e821a2373ecb18f20ea6b93936d4\", table=\"LOG\", type=\"data\"}" timestamp=1689546169683
Jul 17 03:23:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:52.570Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", container=\"coredns\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Async\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546169780
Jul 17 03:23:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:52.891Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_gc_sys_bytes\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"ipmi-exporter\"}" timestamp=1689546171652
Jul 17 03:23:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:53.008Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_dormant\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth736616a4\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546169337
Jul 17 03:23:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:53.087Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_files\", asset=\"pcap-dellr740-dt002\", asset_id=\"33\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda4\", endpoint=\"NC-Gohangout-pcap-dellr740-dt002\", endpoint_id=\"173\", fstype=\"xfs\", module=\"DC-Gohangout\", module_id=\"35\", mountpoint=\"/\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546172666
Jul 17 03:23:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:53.155Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_bytes_total\", asset=\"pcap-dellr740-dt002\", asset_id=\"33\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth2590814\", endpoint=\"NC-Gohangout-pcap-dellr740-dt002\", endpoint_id=\"173\", module=\"DC-Gohangout\", module_id=\"35\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546172666
Jul 17 03:23:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:53.346Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:c0-0000:c0:01.1-0000:c1:00.0-infiniband_mad-umad8.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689546165496
Jul 17 03:23:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:53.477Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-veth736616a4.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689546165496
Jul 17 03:23:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:53.496Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"var-lib-kubelet-pods-05cea850\\\\x2dfebe\\\\x2d4ad9\\\\x2da01e\\\\x2dde2d8c0e8a8b-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2dzgn8l.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689546165496
Jul 17 03:23:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:53.501Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_gc_duration_seconds\", asset=\"pcap-dellr740-dt002\", asset_id=\"33\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"NC-Gohangout-pcap-dellr740-dt002\", endpoint_id=\"173\", module=\"DC-Gohangout\", module_id=\"35\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", quantile=\"0\"}" timestamp=1689546172666
Jul 17 03:23:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:53.537Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens6f1np1\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546163391
Jul 17 03:23:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:53.581Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_scaling_frequency_max_hertz\", asset=\"pcap-dellr740-dt002\", asset_id=\"33\", cpu=\"18\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"NC-Gohangout-pcap-dellr740-dt002\", endpoint_id=\"173\", module=\"DC-Gohangout\", module_id=\"35\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546172666
Jul 17 03:23:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:53.594Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_metadata_resource_version\", asset=\"msh-tsgx12\", asset_id=\"113\", configmap=\"extension-apiserver-authentication\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546168663
Jul 17 03:23:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:53.602Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_rss\", asset=\"msh-tsgx12\", asset_id=\"113\", container=\"ipmi-exporter\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546153470
Jul 17 03:23:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:53.604Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_metadata_generation\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"coredns\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546168663
Jul 17 03:23:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:53.648Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_limit_bytes\", asset=\"msh-tsgx12\", asset_id=\"113\", container=\"telegraf\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689546160589
Jul 17 03:23:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:53.650Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas_unavailable\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"ipmi-exporter\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546168663
Jul 17 03:23:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:53.656Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_packets_total\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens8f1np1\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546163391
Jul 17 03:23:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:53.671Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_commands_total\", asset=\"msh-tsgx12\", asset_id=\"113\", cmd=\"set\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546158659
Jul 17 03:23:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:53.673Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx12\", asset_id=\"113\", cpu=\"61\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546163391
Jul 17 03:23:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:53.694Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_up_changes_total\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"br_dign_s\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546163391
Jul 17 03:23:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:53.695Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dracut-pre-mount.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689546171179
Jul 17 03:23:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:53.714Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"clickhouse_background_pool_task\", asset=\"pcap-dellr740-dt003\", asset_id=\"35\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"NC-Clickhouse-pcap-dellr740-dt003\", endpoint_id=\"85\", module=\"NC-Clickhouse\", module_id=\"29\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546173239
Jul 17 03:23:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:53.721Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_aof_last_cow_size_bytes\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546158659
Jul 17 03:23:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:53.743Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db84\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546158659
Jul 17 03:23:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:53.760Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-sysusers.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"service\"}" timestamp=1689546171179
Jul 17 03:23:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:53.782Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_fifo_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"bond0\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546169337
Jul 17 03:23:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:53.792Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_changes_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth5162d954\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546159409
Jul 17 03:23:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:53.798Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_files\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda2\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", fstype=\"ext4\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/mnt/onie-boot\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546168328
Jul 17 03:23:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:53.853Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_spec_replicas\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"prometheus-alertmanager-6df7f457b6\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546168663
Jul 17 03:23:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:53.927Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_stack_inuse_bytes\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"ipmi-exporter\"}" timestamp=1689546159391
Jul 17 03:23:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:53.937Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx12\", asset_id=\"113\", cpu=\"35\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546163391
Jul 17 03:23:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:53.942Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx08\", asset_id=\"65\", cpu=\"82\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546168328
Jul 17 03:23:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:53.965Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx12\", asset_id=\"113\", cpu=\"74\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546163391
Jul 17 03:23:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:54.002Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_restarts_total\", asset=\"msh-tsgx12\", asset_id=\"113\", container=\"ipmi-exporter\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"ipmi-exporter-6f8464dd94-792kb\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"da81ea3f-d797-4414-9e38-bc69ec182d3b\"}" timestamp=1689546168663
Jul 17 03:23:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:54.008Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db68\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546158659
Jul 17 03:23:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:54.025Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_writes_bytes_total\", asset=\"msh-tsgx12\", asset_id=\"113\", container=\"traefik\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546159873
Jul 17 03:23:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:54.027Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"msh-tsgx08\", asset_id=\"65\", condition=\"Progressing\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"kubernetes-dashboard\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"unknown\"}" timestamp=1689546168950
Jul 17 03:23:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:54.029Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx12\", asset_id=\"113\", container=\"dign-client\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Read\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546148330
Jul 17 03:23:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:54.045Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_last_seen\", asset=\"msh-tsgx12\", asset_id=\"113\", container=\"prometheus-alertmanager\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546160622
Jul 17 03:23:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:54.046Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"var-lib-kubelet-pods-e3fb9886\\\\x2d6958\\\\x2d4504\\\\x2d8536\\\\x2d21d26d231bc8-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2d8zdwv.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689546171179
Jul 17 03:23:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:54.068Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx08\", asset_id=\"65\", cpu=\"61\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546168328
Jul 17 03:23:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:54.069Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_reads_bytes_total\", asset=\"msh-tsgx08\", asset_id=\"65\", container=\"ipmi-exporter\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546151281
Jul 17 03:23:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:54.085Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_discards_transmitted_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"mlx5_1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546169337
Jul 17 03:23:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:54.104Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_bytes_total\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"cadvisor\", interface=\"ens4f0np0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546155676
Jul 17 03:23:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:54.121Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-d05f0b22cf438d3ba8e42a03494e16c18e12d2fd1e570c377d6477a92af3264f.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"scope\"}" timestamp=1689546165496
Jul 17 03:23:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:54.128Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"msh-tsgx08\", asset_id=\"65\", condition=\"Progressing\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"prometheus-alertmanager\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"unknown\"}" timestamp=1689546168950
Jul 17 03:23:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:54.143Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_drop_total\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"virtio_dign_c\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546168328
Jul 17 03:23:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:54.296Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_status_active\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", job_name=\"helm-install-traefik\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546168950
Jul 17 03:23:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:54.298Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx12\", asset_id=\"113\", cpu=\"109\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546163391
Jul 17 03:23:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:54.324Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cpu=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546159409
Jul 17 03:23:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:54.338Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cpu=\"58\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546159409
Jul 17 03:23:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:54.351Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_compressed_total\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens8f1np1\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546163391
Jul 17 03:23:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:54.355Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx12\", asset_id=\"113\", container=\"dign-client\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Async\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546148330
Jul 17 03:23:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:54.373Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"msh-tsgx12\", asset_id=\"113\", container=\"prometheus-pushgateway\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"cadvisor\", failure_type=\"pgfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"hierarchy\"}" timestamp=1689546147276
Jul 17 03:23:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:54.377Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_physical_state_id\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_7\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546163391
Jul 17 03:23:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:54.382Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_metadata_generation\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"ipmi-exporter-6f8464dd94\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546162058
Jul 17 03:23:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:54.389Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_protocol_type\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens4f1np1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546159409
Jul 17 03:23:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:54.397Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-8ba148c1\\\\x2df25f\\\\x2d4fa4\\\\x2d82b3\\\\x2d03b9d5a8ace6-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2d8fdz6.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689546171179
Jul 17 03:23:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:54.416Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens4f0np0\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546163391
Jul 17 03:23:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:54.534Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db181\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546170108
Jul 17 03:23:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:54.561Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cpu=\"13\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546159409
Jul 17 03:23:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:54.562Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cpu=\"46\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546159409
Jul 17 03:23:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:54.569Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_last_seen\", asset=\"pcap-tsgx07\", asset_id=\"85\", container=\"firewall\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689546170632
Jul 17 03:23:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:54.578Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cpu=\"66\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546159409
Jul 17 03:23:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:54.611Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"initrd-cleanup.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689546160884
Jul 17 03:23:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:54.693Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cpu=\"61\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546159409
Jul 17 03:23:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:54.758Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"ostree-remount.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"oneshot\", type=\"service\"}" timestamp=1689546171179
Jul 17 03:23:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:56.486Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"122\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546165883
Jul 17 03:23:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:56.838Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dracut-shutdown.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689546163372
Jul 17 03:23:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:56.946Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_virtual_memory_bytes\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-journald.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689546163372
Jul 17 03:23:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:56.964Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2dlabel-ONIE\\\\x2dBOOT.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689546163372
Jul 17 03:23:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:57.036Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"24\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546165883
Jul 17 03:23:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:57.058Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db126\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546166475
Jul 17 03:23:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:57.074Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"tmpfs\", name=\"run-k3s-containerd-io.containerd.grpc.v1.cri-sandboxes-dda55d28ac191361ea9a19f0d66d4356bac0c1b7f887b804fbd49d043a7b9547-shm.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689546163372
Jul 17 03:23:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:57.119Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_init_container_status_terminated\", asset=\"pcap-tsgx03\", asset_id=\"73\", container=\"tsg-dign-init\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"dign-server-gvdww\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"8a18b7c9-5f34-4084-b472-fc3356545d52\"}" timestamp=1689546173389
Jul 17 03:23:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:57.140Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_replicas\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"local-path-provisioner-957fdf8bc\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546173389
Jul 17 03:23:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:57.159Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_timex_tick_seconds\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546165883
Jul 17 03:23:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:57.173Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_resident_memory_bytes\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"atd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689546163372
Jul 17 03:23:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:57.284Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_rdb_last_save_timestamp_seconds\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546166475
Jul 17 03:23:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:23:57.590Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_terminated\", asset=\"pcap-tsgx03\", asset_id=\"73\", container=\"prometheus-server-configmap-reload\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-server-6ff784d758-2bs79\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"d1430253-8c7e-449d-9f2a-dd552c402bae\"}" timestamp=1689546173389
Jul 17 03:24:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:00.153Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"39\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546169083
Jul 17 03:24:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:00.180Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_compressed_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens3f1np1\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546169083
Jul 17 03:24:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:00.569Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_status_completion_time\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", job_name=\"helm-install-systemd-exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546167425
Jul 17 03:24:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:00.965Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_bytes_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens2f0np0\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546169083
Jul 17 03:24:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:01.044Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_fifo_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"br_dign_c\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546169083
Jul 17 03:24:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:01.882Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"msh-tsgx01\", asset_id=\"51\", container=\"dign-server-web\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"cadvisor\", failure_type=\"pgfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"hierarchy\"}" timestamp=1689546162988
Jul 17 03:24:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:01.974Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_spec_replicas\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-1-firewall-6c6f4bc877\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546167425
Jul 17 03:24:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:01.982Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-virtual-net-br_dign_s.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689546164879
Jul 17 03:24:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:02.550Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_colls_total\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens4f3np3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546178318
Jul 17 03:24:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:02.807Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_status_succeeded\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", job_name=\"helm-install-cm-policy-local-cache\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546171412
Jul 17 03:24:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:02.937Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx02\", asset_id=\"63\", cpu=\"86\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546178318
Jul 17 03:24:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:03.006Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_name_assign_type\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"eno2\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546178318
Jul 17 03:24:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:03.107Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"prometheus_target_sync_length_seconds\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", instance=\"10.10.20.159:10090\", quantile=\"0.05\", scrape_job=\"asset_ping_117\", type=\"global\"}" timestamp=1689546182989
Jul 17 03:24:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:03.268Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"lo\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546178318
Jul 17 03:24:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:03.270Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"prometheus_target_sync_length_seconds\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", instance=\"10.10.20.159:10090\", quantile=\"0.05\", scrape_job=\"321\", type=\"global\"}" timestamp=1689546182989
Jul 17 03:24:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:03.290Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_mspan_sys_bytes\", asset=\"pcap-dellr740-dt002\", asset_id=\"33\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"OLAP-Node-Exporter-pcap-dellr740-dt002\", endpoint_id=\"253\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546182497
Jul 17 03:24:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:03.534Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_address_not_ready\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_endpoint=\"prometheus-alertmanager\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546171412
Jul 17 03:24:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:03.579Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", phase=\"Pending\", pod=\"prometheus-server-6ff784d758-nlwkc\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"f11152e9-dd9a-4d07-9e50-6163396c01f6\"}" timestamp=1689546171412
Jul 17 03:24:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:03.621Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_load_average_10s\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689546167141
Jul 17 03:24:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:03.867Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_info\", asset=\"msh-tsgx02\", asset_id=\"63\", created_by_kind=\"Job\", created_by_name=\"helm-install-cm-policy-local-cache\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", host_ip=\"10.10.20.162\", host_network=\"false\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", node=\"localhost\", nz_agent_id=\"3\", pod=\"helm-install-cm-policy-local-cache-tv4ds\", pod_ip=\"10.42.0.158\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"a36b4256-35de-4113-98f6-93ad55836cde\"}" timestamp=1689546171412
Jul 17 03:24:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:03.917Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"99\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546178318
Jul 17 03:24:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:04.026Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_info\", asset=\"msh-tsgx02\", asset_id=\"63\", container=\"dign-server-dns\", container_id=\"containerd://e8b341f432e3d8404c563608abb9a5256fa76ccc78a35bcdfd5d4da405ec243c\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", image=\"docker.io/library/dign-server-dns:latest\", image_id=\"sha256:6987ac49c2d264e9289120b41e4a48536092ce6fbe41c981cdd4ad115feae8a4\", image_spec=\"dign-server-dns:latest\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"dign-server-2lzxm\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"5bf242d5-47db-451d-bd36-9e1874e24271\"}" timestamp=1689546171412
Jul 17 03:24:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:04.494Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db40\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546169269
Jul 17 03:24:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:04.516Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_swap\", asset=\"msh-tsgx02\", asset_id=\"63\", container=\"dign-server-dns\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546166637
Jul 17 03:24:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:04.544Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_disk_writes_merged_total\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"sdb\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-dt001\", endpoint_id=\"225\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546182987
Jul 17 03:24:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:04.561Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas_unavailable\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"cm-policy-local-cache-redis-master\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546171412
Jul 17 03:24:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:04.590Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_max_fds\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"snmpd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689546168447
Jul 17 03:24:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:04.602Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_files\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"shm\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/8ed749028134fa72c1bd5589c09d044c0e5e7242ae7cd91fd54311b241ac15b1/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546178318
Jul 17 03:24:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:04.641Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_fifo_total\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"br-851e3f3d436f\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-dt001\", endpoint_id=\"225\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546182987
Jul 17 03:24:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:04.655Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"rc-local.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689546168447
Jul 17 03:24:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:04.676Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-43d98779\\\\x2d0eca\\\\x2d4764\\\\x2dab37\\\\x2da5e20cb923be-volume\\\\x2dsubpaths-telegraf-telegraf\\\\x2dsecurity-0.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689546168447
Jul 17 03:24:05 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:05.163Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"prometheus_sd_linode_failures_total\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", instance=\"10.10.20.159:10090\", type=\"global\"}" timestamp=1689546182989
Jul 17 03:24:05 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:05.173Z caller=dedupe.go:112 component=remote level=info remote_name=52db6f url=http://10.10.10.159:10090/nz-agent/prometheus/proxy/api/v1/write msg="Dropped sample for series that was not explicitly dropped via relabelling" ref=4436650
Jul 17 03:24:05 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:05.176Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"prometheus_target_sync_length_seconds_sum\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", instance=\"10.10.20.159:10090\", scrape_job=\"443\", type=\"global\"}" timestamp=1689546182989
Jul 17 03:24:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:06.377Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_scaling_frequency_max_hertz\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", cpu=\"1\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Spark-msh-dellr740-olap003\", endpoint_id=\"13\", module=\"NC-Spark\", module_id=\"47\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546186091
Jul 17 03:24:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:06.694Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pnp0-00:04-tty-ttyS1.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689546173582
Jul 17 03:24:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:06.838Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"63\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546183910
Jul 17 03:24:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:06.887Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_compressed_total\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens8f0np0\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546183910
Jul 17 03:24:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:06.958Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failcnt\", asset=\"twa-tsgx01\", asset_id=\"91\", container=\"metrics-server\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546169523
Jul 17 03:24:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:06.970Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_complete\", asset=\"twa-tsgx01\", asset_id=\"91\", condition=\"false\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", job_name=\"helm-install-mrzcpd-exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546181227
Jul 17 03:24:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:07.046Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2dpartuuid-41f6cff2\\\\x2dfc14\\\\x2d44d5\\\\x2d8fc8\\\\x2dbe6b6ea3a3e7.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689546173582
Jul 17 03:24:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:07.108Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-707e9bd65cf854d2199bb63803f542c9c070306376188b911495f4595922b2ae-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689546173582
Jul 17 03:24:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:07.528Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Read\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546181554
Jul 17 03:24:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:07.992Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_working_set_bytes\", asset=\"twa-tsgx01\", asset_id=\"91\", container=\"kube-multus\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546179771
Jul 17 03:24:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:08.081Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cpu=\"42\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546183910
Jul 17 03:24:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:08.107Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_errors_total\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"cadvisor\", interface=\"ens4f2np2\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546181096
Jul 17 03:24:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:08.270Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"nvmefc-boot-connections.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"service\"}" timestamp=1689546173582
Jul 17 03:24:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:08.403Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"clickhouse_block_queue_time_sda\", asset=\"msh-dellr740-olap002\", asset_id=\"13\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Clickhouse-msh-dellr740-olap002\", endpoint_id=\"73\", module=\"NC-Clickhouse\", module_id=\"29\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546187761
Jul 17 03:24:09 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:09.671Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_lastMajorCompactionAge\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"firewall_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"8a4c8c9df530241b5d7eaf5c4ed6ed9b\", table=\"index_time_firewall_hos_bucket\", type=\"index\"}" timestamp=1689546188804
Jul 17 03:24:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:10.073Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_compactionsFailedCount\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"troubleshooting_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"04d1ad2a5fa32c13020ced7fe677796a\", table=\"index_filename_troubleshooting_hos_bucket\", type=\"index\"}" timestamp=1689546188804
Jul 17 03:24:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:10.186Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_compactionsCompletedCount\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"session_record_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"2fa4fd289d02170d9d7aaa6005053c98\", table=\"index_time_session_record_hos_bucket\", type=\"index\"}" timestamp=1689546188804
Jul 17 03:24:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:10.373Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_filteredReadRequestCount\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"troubleshooting_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"31370399349cf9df0995c4e2392ab722\", table=\"index_partfile_troubleshooting_hos_bucket\", type=\"index\"}" timestamp=1689546188804
Jul 17 03:24:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:10.507Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_maxFlushQueueSize\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"211c7f6f7154f8cf14f1db5b8e1157c5\", table=\"knowledge_base_hos_bucket\", type=\"data\"}" timestamp=1689546188804
Jul 17 03:24:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:10.533Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Increment_max\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Server\"}" timestamp=1689546188804
Jul 17 03:24:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:11.289Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_deleteCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", namespace=\"SYSTEM\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"71f969630421eb2700553877cf2f18d9\", table=\"LOG\", type=\"data\"}" timestamp=1689546190147
Jul 17 03:24:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:11.413Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_4_metric_majorCompactionOutputSize_99th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546190147
Jul 17 03:24:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:12.112Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_session_record_hos_bucket_metric_majorCompactionInputSize_99_9th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546190147
Jul 17 03:24:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:12.313Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_STATS_metric_putBatchTime_min\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546190147
Jul 17 03:24:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:12.328Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_incrementCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"session_record_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"9fbee1d80ed21c8dac9cbbe23a6de52e\", table=\"index_filename_session_record_hos_bucket\", type=\"index\"}" timestamp=1689546190147
Jul 17 03:24:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:12.414Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_lastMajorCompactionAge\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"session_record_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"fe5edc011eaddb1eb1c7961140d3dbc5\", table=\"index_filename_session_record_hos_bucket\", type=\"index\"}" timestamp=1689546190147
Jul 17 03:24:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:12.416Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_numBytesCompactedCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"rtp_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"afda67220314ac681ee8f19c890bcc36\", table=\"index_partfile_rtp_hos_bucket\", type=\"index\"}" timestamp=1689546190147
Jul 17 03:24:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:12.430Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_session_record_hos_bucket_metric_compactionInputFileCount_median\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546190147
Jul 17 03:24:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:12.574Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_default_metric_appendTime_99_9th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546190147
Jul 17 03:24:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:12.742Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_tsg_table_report_result_metric_majorCompactionTime_min\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546190147
Jul 17 03:24:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:12.763Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_tsg_table_report_result_metric_majorCompactionInputFileCount_median\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546190147
Jul 17 03:24:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:12.783Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_replicaid\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"proxy_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"d5e2c7c3871e9be6227122cc1dcaa828\", table=\"index_partfile_proxy_hos_bucket\", type=\"index\"}" timestamp=1689546190147
Jul 17 03:24:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:15.416Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_owner\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", owner_is_controller=\"true\", owner_kind=\"ReplicaSet\", owner_name=\"dashboard-metrics-scraper-5b94897dc9\", pod=\"dashboard-metrics-scraper-5b94897dc9-n6m9s\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"d75737d8-d596-4b86-8834-e9b0de808309\"}" timestamp=1689546188076
Jul 17 03:24:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:15.447Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"104\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546181230
Jul 17 03:24:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:15.454Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-100d4d621a68c3f2412a9ad2bc1fe4119c42a11e117ef1fc9ed869fd813d94c6-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689546187046
Jul 17 03:24:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:15.483Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"94\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546181230
Jul 17 03:24:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:15.497Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dracut-shutdown.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"service\"}" timestamp=1689546187046
Jul 17 03:24:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:15.559Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-a1939361db600e2d361cb6066288813fd93c634b22d797abfbbf0b04ce241276-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689546187046
Jul 17 03:24:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:15.629Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Unknown\", pod=\"coredns-65dfb4fcf4-ss5kl\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"d909c7f6-8616-4c21-913b-225bb207a2ea\"}" timestamp=1689546188076
Jul 17 03:24:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:15.644Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"106\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546181230
Jul 17 03:24:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:15.657Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_cpu_period\", asset=\"twa-tsgx07\", asset_id=\"103\", container=\"telegraf-security\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689546189150
Jul 17 03:24:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:15.681Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"47\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546181230
Jul 17 03:24:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:15.812Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Running\", pod=\"helm-install-prometheus-vmmrn\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"9a25cfd7-ced4-44df-b397-80f3bbb69a79\"}" timestamp=1689546188076
Jul 17 03:24:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:15.815Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_resident_memory_bytes\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"tuned.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689546187046
Jul 17 03:24:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:15.831Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"nsfs\", name=\"run-netns-cni\\\\x2df3251bc8\\\\x2d8826\\\\x2d670a\\\\x2d05ae\\\\x2ddf44f9cd99c1.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689546187046
Jul 17 03:24:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:15.864Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546181230
Jul 17 03:24:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:16.007Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_type\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.tsg-traffic-engine-numa-1.v2\", service=\"prometheus-kube-state-metrics\", type=\"helm.sh/release.v1\"}" timestamp=1689546188076
Jul 17 03:24:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:16.020Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"48\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546181230
Jul 17 03:24:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:16.038Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"phy_tx_bits_total\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", dev=\"ens1f0np0\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"mrzcpd-exporter\"}" timestamp=1689546186692
Jul 17 03:24:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:16.149Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_packets_total\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens7f1np1\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546181230
Jul 17 03:24:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:16.297Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"twa-tsgx07\", asset_id=\"103\", container=\"local-path-provisioner\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"sleeping\"}" timestamp=1689546180116
Jul 17 03:24:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:16.308Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_vl15_dropped_total\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_6\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546181230
Jul 17 03:24:18 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:18.784Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"clickhouse_temperature_coretemp__package_id_0\", asset=\"twa-dellr740-dt003\", asset_id=\"41\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"NC-Clickhouse-twa-dellr740-dt003\", endpoint_id=\"91\", module=\"NC-Clickhouse\", module_id=\"29\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546198301
Jul 17 03:24:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:19.338Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"clickhouse_os_soft_irq_time_cpu37\", asset=\"twa-dellr740-dt003\", asset_id=\"41\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"NC-Clickhouse-twa-dellr740-dt003\", endpoint_id=\"91\", module=\"NC-Clickhouse\", module_id=\"29\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546198301
Jul 17 03:24:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:19.442Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_errors_total\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"cadvisor\", interface=\"eno1\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546189505
Jul 17 03:24:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:19.456Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_complete\", asset=\"twa-tsgx06\", asset_id=\"101\", condition=\"false\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", job_name=\"helm-install-systemd-exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546193949
Jul 17 03:24:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:19.667Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_created\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"dign-client-llnpb\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"78246b9b-6c2f-4139-8e55-13e52335743d\"}" timestamp=1689546193949
Jul 17 03:24:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:19.695Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_errs_total\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens4f3np3\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546197612
Jul 17 03:24:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:19.712Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_threads\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"ipmi-exporter\"}" timestamp=1689546192714
Jul 17 03:24:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:19.737Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"twa-tsgx06\", asset_id=\"101\", condition=\"false\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"cm-policy-local-cache-redis-master-69967b5879-skrxn\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"4ce0222b-34a2-45cf-bbe5-fd12c5e4d780\"}" timestamp=1689546193949
Jul 17 03:24:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:19.738Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"18\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546197612
Jul 17 03:24:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:19.780Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-3a00f506971912e91a0f2421a4cd5f23450e9b733c343cbedf66427667057151.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"scope\"}" timestamp=1689546184356
Jul 17 03:24:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:19.782Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"121\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546197612
Jul 17 03:24:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:19.873Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-virtual-net-veth48720f3f.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689546184356
Jul 17 03:24:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:20.132Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-2af298e7d367e5f0afddf41c74eda98caac9331a2bde64a8df68bebe065ed13c.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"scope\"}" timestamp=1689546184356
Jul 17 03:24:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:20.164Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-pod5f8b23e9_8830_4b0a_9d61_06b46edd96f0.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"slice\"}" timestamp=1689546184356
Jul 17 03:24:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:20.376Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_spec_replicas\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"tsg-traffic-engine-numa-1-firewall\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546193949
Jul 17 03:24:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:20.698Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_virtual_memory_max_bytes\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"mrzcpd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689546190657
Jul 17 03:24:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:20.726Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_drop_total\", asset=\"twa-tsgx12\", asset_id=\"121\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethc8e06791\", endpoint=\"TSG-OS-twa-tsgx12\", endpoint_id=\"463\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546192766
Jul 17 03:24:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:20.771Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"61\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546197612
Jul 17 03:24:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:20.780Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_multicast_packets_received_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"mlx5_0\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546188263
Jul 17 03:24:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:20.790Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:00-0000:00:01.1-0000:01:00.0-net-ens1f0np0.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689546184356
Jul 17 03:24:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:20.819Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-virtual-net-vethd8c52418.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689546184356
Jul 17 03:24:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:20.838Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_annotations\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", job_name=\"helm-install-traefik\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546193949
Jul 17 03:24:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:20.909Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_bytes_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth3417e171\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546188263
Jul 17 03:24:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:20.936Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"100\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546188263
Jul 17 03:24:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:20.952Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"77\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546188263
Jul 17 03:24:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:21.066Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-a74e5213\\\\x2d995f\\\\x2d4251\\\\x2da2d7\\\\x2d39614c7890a8-volume\\\\x2dsubpaths-config\\\\x2dpath-dign\\\\x2dserver\\\\x2ddns-1.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689546184356
Jul 17 03:24:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:21.098Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"tmpfs\", name=\"var-lib-kubelet-pods-eafb2c24\\\\x2df0a2\\\\x2d46e9\\\\x2d837e\\\\x2de7db0ce25c25-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2dw2n2l.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689546190657
Jul 17 03:24:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:21.205Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_errors_received_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"mlx5_4\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546188263
Jul 17 03:24:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:21.287Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_spec_type\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"mrzcpd-exporter\", type=\"ClusterIP\", uid=\"df277af0-d1b5-4d1e-a7bf-3b0765fa5ef7\"}" timestamp=1689546193949
Jul 17 03:24:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:21.951Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_last_seen\", asset=\"pcap-tsgx10\", asset_id=\"89\", container=\"exporter-merger\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546195757
Jul 17 03:24:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:22.046Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_info\", address=\"1a:ae:e0:1f:d4:f2\", asset=\"pcap-tsgx10\", asset_id=\"89\", broadcast=\"ff:ff:ff:ff:ff:ff\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"br_dign_s\", duplex=\"unknown\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operstate=\"up\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546188263
Jul 17 03:24:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:23.959Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-2fc274a0\\\\x2d8e42\\\\x2d474b\\\\x2db1fe\\\\x2da285bc291240-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2d8sxjg.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689546198416
Jul 17 03:24:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:24.007Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx09\", asset_id=\"55\", cpu=\"126\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546194790
Jul 17 03:24:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:24.107Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_gc_cpu_fraction\", asset=\"msh-dellr740-olap004\", asset_id=\"17\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Spark-msh-dellr740-olap004\", endpoint_id=\"15\", module=\"NC-Spark\", module_id=\"47\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546202254
Jul 17 03:24:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:24.159Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db248\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546197019
Jul 17 03:24:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:24.189Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_address_assign_type\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"eno36np3\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546200160
Jul 17 03:24:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:24.207Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-virtual-net-veth7ba955f4.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689546191102
Jul 17 03:24:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:24.298Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"82\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546200160
Jul 17 03:24:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:24.317Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_avail_bytes\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"shm\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/a675f3d38358a7aaabb7854e5e18bf454618c3432aec1414f1ea31c97a2d9987/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546200160
Jul 17 03:24:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:24.350Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_virtual_memory_max_bytes\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"rdma-ndd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689546191102
Jul 17 03:24:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:24.352Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.grpc.v1.cri-sandboxes-b68f6826fbbf6c5ee858974ef0ea742a94d6b4a83f1bf5bfbdaca46b6f6d374a-shm.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689546191102
Jul 17 03:24:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:24.509Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_last_seen\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"kube-multus\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546203450
Jul 17 03:24:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:24.512Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_spec_paused\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"ipmi-exporter\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546200068
Jul 17 03:24:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:24.557Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_created\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.prometheus.v1\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546200068
Jul 17 03:24:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:24.662Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"pcap-tsgx12\", asset_id=\"117\", condition=\"unknown\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-server-6ff784d758-wz5rs\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"ab6b1591-cfaa-4964-8fcc-a861d0862792\"}" timestamp=1689546191122
Jul 17 03:24:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:24.679Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"104\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546200160
Jul 17 03:24:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:24.789Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"msh-tsgx09\", asset_id=\"55\", cpu=\"83\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546194790
Jul 17 03:24:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:24.873Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"msh-tsgx09\", asset_id=\"55\", cpu=\"99\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546194790
Jul 17 03:24:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:25.014Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:80-0000:80:01.1-0000:81:00.0-host0-target0:2:0-0:2:0:0-block-sda-sda2.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689546191102
Jul 17 03:24:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:25.059Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_ports\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_endpoint=\"traefik\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", port_name=\"websecure\", port_number=\"8443\", port_protocol=\"TCP\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546200068
Jul 17 03:24:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:25.077Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_compressed_total\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"vethb06b3a71\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546194790
Jul 17 03:24:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:25.127Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-burstable-poda6dafded_0d01_479f_906c_089305c5928a.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"slice\"}" timestamp=1689546191102
Jul 17 03:24:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:25.156Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"var-lib-kubelet-pods-78d32c4f\\\\x2d349d\\\\x2d455c\\\\x2d932b\\\\x2db75d141ec6e9-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2dl2gp9.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689546191102
Jul 17 03:24:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:25.335Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_commands_rejected_calls_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cmd=\"zscore\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546201795
Jul 17 03:24:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:25.352Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Tcp_Link_New\", app_name=\"sapp4\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", vsysId=\"1\"}" timestamp=1689546192900
Jul 17 03:24:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:25.454Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-1f66a3cd08d7e75bc6dbae54e0bd46d50e7e32f36898b4f34604897cd1440727.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"scope\"}" timestamp=1689546191102
Jul 17 03:24:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:25.468Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:20-0000:20:03.1-0000:21:00.0-net-ens2f0np0.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689546191102
Jul 17 03:24:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:25.526Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_state_started\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"prometheus-alertmanager\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-alertmanager-6df7f457b6-2h45l\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"b60f186f-e535-4016-adab-bd4c742131b8\"}" timestamp=1689546191122
Jul 17 03:24:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:25.580Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_compressed_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"bond0\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546200160
Jul 17 03:24:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:25.695Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_labels\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-1-firewall-67c49c4495\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546191122
Jul 17 03:24:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:25.977Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"27\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546195810
Jul 17 03:24:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:25.982Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens3f1np1\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546195810
Jul 17 03:24:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:26.022Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_name_assign_type\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens6f0np0\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546195810
Jul 17 03:24:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:26.026Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_sockstat_UDP_inuse\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546195810
Jul 17 03:24:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:26.056Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_terminated_reason\", asset=\"twa-tsgx09\", asset_id=\"107\", container=\"helm\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-configmap-reloader-hq5hk\", project=\"TSG-OS\", project_id=\"7\", reason=\"Completed\", service=\"prometheus-kube-state-metrics\", uid=\"cd026eda-ed7f-456d-af88-a83999108711\"}" timestamp=1689546198744
Jul 17 03:24:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:26.093Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"23\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546195810
Jul 17 03:24:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:26.139Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_discards_transmitted_total\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_2\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546195810
Jul 17 03:24:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:26.273Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_packets_total\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethf2a1e699\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546195810
Jul 17 03:24:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:26.329Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_receive_remote_physical_errors_total\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_4\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546195810
Jul 17 03:24:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:26.459Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_entropy_available_bits\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546195810
Jul 17 03:24:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:26.506Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_size_bytes\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"shm\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/ca9754331bf01fc4c5e0868d11b20357bf091bb1a8145499cb20ece7e612f25a/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546195810
Jul 17 03:24:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:26.538Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"paths.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"target\"}" timestamp=1689546193907
Jul 17 03:24:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:27.073Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_errors_received_total\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_3\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546195810
Jul 17 03:24:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:27.079Z caller=dedupe.go:112 component=remote level=error remote_name=52db6f url=http://10.10.10.159:10090/nz-agent/prometheus/proxy/api/v1/write msg="non-recoverable error" count=1000 exemplarCount=0 err="server returned HTTP status 400 : out of order sample"
Jul 17 03:24:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:27.250Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_packets_transmitted_total\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_4\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546195810
Jul 17 03:24:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:27.406Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"58\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546202882
Jul 17 03:24:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:27.427Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-ce676e45\\\\x2d7527\\\\x2d495c\\\\x2d945c\\\\x2dc9fa2e1248cd-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2dmhtnk.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689546194114
Jul 17 03:24:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:27.505Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas_ready\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"prometheus-alertmanager\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546195682
Jul 17 03:24:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:27.568Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"atd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"service\"}" timestamp=1689546194114
Jul 17 03:24:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:27.656Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link_mode\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth2e13d535\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546202882
Jul 17 03:24:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:27.725Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_rate_bytes_per_second\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_4\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546195810
Jul 17 03:24:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:27.769Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2did-scsi\\\\x2dSDELL_PERC_H345_Adp_00b55a48ca12f7252c00b94683f0a7ce\\\\x2dpart1.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689546193907
Jul 17 03:24:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:27.777Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", phase=\"Failed\", pod=\"prometheus-kube-state-metrics-6b768c7bf8-c5glh\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"1d1df77f-66ac-4156-8ea3-66e8a287970d\"}" timestamp=1689546198744
Jul 17 03:24:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:27.822Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"msh-tsgx10\", asset_id=\"69\", condition=\"true\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"configmap-reloader-6c55bbc49f-w8sx6\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"dd88c173-8b8c-4cdc-afaf-b521eb296cc1\"}" timestamp=1689546195682
Jul 17 03:24:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:27.851Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db228\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546199446
Jul 17 03:24:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:27.905Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"6\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546195810
Jul 17 03:24:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:27.924Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_node_status_allocatable\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", node=\"localhost\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", resource=\"ephemeral_storage\", service=\"prometheus-kube-state-metrics\", unit=\"byte\"}" timestamp=1689546195682
Jul 17 03:24:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:27.961Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_complete\", asset=\"twa-tsgx09\", asset_id=\"107\", condition=\"false\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", job_name=\"helm-install-traefik\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546198744
Jul 17 03:24:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:27.983Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_state_started\", asset=\"twa-tsgx09\", asset_id=\"107\", container=\"telegraf\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", pod=\"tsg-traffic-engine-numa-1-firewall-69dcc646dd-q9d4z\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"16605650-cdf7-400d-9016-8c2dbe1159c9\"}" timestamp=1689546198744
Jul 17 03:24:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:28.153Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_commands_rejected_calls_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cmd=\"expire\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546199446
Jul 17 03:24:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:28.471Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_excessive_buffer_overrun_errors_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_7\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546202882
Jul 17 03:24:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:28.486Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"91\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546202882
Jul 17 03:24:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:28.695Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_daemonset_labels\", asset=\"msh-tsgx10\", asset_id=\"69\", daemonset=\"svclb-traefik-dbdae63f\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546195682
Jul 17 03:24:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:28.731Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_terminated\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"metrics\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"cm-policy-local-cache-redis-master-69967b5879-fpcq2\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"78ab1f97-9ecb-47ed-ae81-048055fb4556\"}" timestamp=1689546195682
Jul 17 03:24:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:28.769Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"cortex_kv_request_duration_seconds_bucket\", agent_name=\"msh-kvm-nz001\", application=\"loki\", datacenter=\"MSH\", instance=\"127.0.0.1:13100\", kv_name=\"distributor-lifecycler\", le=\"+Inf\", operation=\"CAS\", role=\"primary\", status_code=\"200\", type=\"memberlist\"}" timestamp=1689546208327
Jul 17 03:24:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:28.856Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db145\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546199446
Jul 17 03:24:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:28.868Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db163\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546199446
Jul 17 03:24:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:28.881Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_type\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"chart-values-traefik\", service=\"prometheus-kube-state-metrics\", type=\"Opaque\"}" timestamp=1689546195682
Jul 17 03:24:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:31.254Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"nginx_vts_main_shm_usage_bytes\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"Galaxy-hos-nginx-msh-dellr740-dt001\", endpoint_id=\"201\", module=\"Galaxy-hos-nginx\", module_id=\"13\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", shared=\"max_size\"}" timestamp=1689546210872
Jul 17 03:24:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:31.257Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"41\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546200939
Jul 17 03:24:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:31.289Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_state_id\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_1\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546200939
Jul 17 03:24:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:31.355Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_ready_replicas\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"local-path-provisioner-957fdf8bc\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546204068
Jul 17 03:24:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:31.374Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"2\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546200939
Jul 17 03:24:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:31.393Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_speed_bytes\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens4f0np0\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546200939
Jul 17 03:24:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:31.393Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2dpath-pci\\\\x2d0000:81:00.0\\\\x2dscsi\\\\x2d0:2:0:0\\\\x2dpart4.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689546201015
Jul 17 03:24:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:31.411Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"21\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546200939
Jul 17 03:24:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:31.464Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cryptsetup-pre.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"target\"}" timestamp=1689546206907
Jul 17 03:24:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:31.515Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_labels\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_endpoint=\"prometheus-server\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546210436
Jul 17 03:24:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:31.537Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_compressed_total\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens8f0np0\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546199093
Jul 17 03:24:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:31.552Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_frame_total\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth67ec28a7\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546199093
Jul 17 03:24:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:31.582Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_complete\", asset=\"twa-tsgx03\", asset_id=\"95\", condition=\"unknown\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", job_name=\"helm-install-traefik-crd\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546210436
Jul 17 03:24:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:31.585Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"msh-tsgx05\", asset_id=\"67\", container=\"dashboard-metrics-scraper\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"iowaiting\"}" timestamp=1689546189555
Jul 17 03:24:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:31.661Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db212\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546206438
Jul 17 03:24:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:31.672Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-virtual-net-veth23aaa632.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689546206907
Jul 17 03:24:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:31.726Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_dormant\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth497f7384\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546200791
Jul 17 03:24:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:31.768Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-c9b0dc64\\\\x2ded75\\\\x2d445b\\\\x2da713\\\\x2dada2b43ce324-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2dq6wdv.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689546202644
Jul 17 03:24:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:31.779Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"cadvisor_version_info\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", kernelVersion=\"5.17.15-1.el8.x86_64\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", osVersion=\"TSG-OS v23.07-rc1-079e964 (TSGXNXR620G40R01P0906)\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546202009
Jul 17 03:24:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:31.805Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_threads_max\", asset=\"msh-tsgx05\", asset_id=\"67\", container=\"mrapm-device\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546187120
Jul 17 03:24:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:31.964Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-veth244c5093.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689546206907
Jul 17 03:24:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:32.044Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"38\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546199093
Jul 17 03:24:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:32.050Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_reads_total\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"firewall\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689546198659
Jul 17 03:24:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:32.068Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_info\", address=\"08:c0:eb:53:c6:dd\", asset=\"msh-tsgx11\", asset_id=\"111\", broadcast=\"ff:ff:ff:ff:ff:ff\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens2f1np1\", duplex=\"unknown\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operstate=\"down\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546200791
Jul 17 03:24:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:32.101Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"9\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546199093
Jul 17 03:24:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:32.132Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"83\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546200939
Jul 17 03:24:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:32.148Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"eno33np0\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546200939
Jul 17 03:24:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:32.219Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"86\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546200939
Jul 17 03:24:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:32.236Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_info\", address=\"b8:ce:f6:72:b2:14\", asset=\"msh-tsgx05\", asset_id=\"67\", broadcast=\"ff:ff:ff:ff:ff:ff\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens7f0np0\", duplex=\"unknown\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operstate=\"down\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546200939
Jul 17 03:24:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:32.352Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"shm\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/e41815df358d5e53d4c4527c647f58609f01f0d5beaa493c16a5e9f97a73fc72/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546200791
Jul 17 03:24:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:32.584Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_other_sys_bytes\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546199093
Jul 17 03:24:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:32.617Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_start_time\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"svclb-traefik-acf36120-2ckwj\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"14f66a9f-85bf-4104-ac3f-c15dda542ad5\"}" timestamp=1689546210436
Jul 17 03:24:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:32.642Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_hwmon_sensor_label\", asset=\"twa-tsgx03\", asset_id=\"95\", chip=\"pci0000:00_0000:00:18_3\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", label=\"tccd6\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", sensor=\"temp8\", service=\"prometheus-node-exporter\"}" timestamp=1689546199093
Jul 17 03:24:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:32.766Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_load1\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546200605
Jul 17 03:24:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:32.791Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_flags\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth50f7b1ca\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546200605
Jul 17 03:24:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:32.806Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_ready_replicas\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"metrics-server-54dc485875\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546199276
Jul 17 03:24:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:32.811Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-d880e4adf6407044b8dfb78ba2d55193308659f0f6fbc3390785a72c29ec160f.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"scope\"}" timestamp=1689546202644
Jul 17 03:24:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:32.831Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_replicas\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"mrzcpd-exporter-5cf7f967cc\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546199276
Jul 17 03:24:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:33.142Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"123\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546200605
Jul 17 03:24:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:33.161Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_state_started\", asset=\"twa-tsgx03\", asset_id=\"95\", container=\"helm\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-tsg-traffic-engine-numa-1-vnr97\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"655855cf-bcb1-4cce-bea6-d09a402d65d3\"}" timestamp=1689546210436
Jul 17 03:24:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:33.172Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-prometheus-mshnr\", project=\"TSG-OS\", project_id=\"7\", reason=\"Evicted\", service=\"prometheus-kube-state-metrics\", uid=\"6fb2bd33-bd15-4e41-9452-847ce05e8c34\"}" timestamp=1689546210436
Jul 17 03:24:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:33.741Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_running\", asset=\"twa-tsgx05\", asset_id=\"99\", container=\"firewall\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", pod=\"tsg-traffic-engine-numa-0-firewall-7698d7c499-8zkwn\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"9982afd0-5736-4dce-b764-2f7b0c30831a\"}" timestamp=1689546199894
Jul 17 03:24:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:33.818Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_rss\", asset=\"twa-tsgx05\", asset_id=\"99\", container=\"traefik\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546187777
Jul 17 03:24:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:33.832Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-9fec12b63663a4b4c3edc8da0ea94d0f53bed6ca984ae78b45fed0d7e9985c3a.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"scope\"}" timestamp=1689546207691
Jul 17 03:24:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:33.860Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_scrape_collector_success\", asset=\"twa-tsgx05\", asset_id=\"99\", collector=\"arp\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546200605
Jul 17 03:24:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:33.876Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"36\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546200605
Jul 17 03:24:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:34.108Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_created\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-headless\", uid=\"964c9957-7e9b-46d0-9c50-cc6765d99b8c\"}" timestamp=1689546199894
Jul 17 03:24:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:34.133Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_load5\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546200605
Jul 17 03:24:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:34.286Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_spec_replicas\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"prometheus-server\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546199894
Jul 17 03:24:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:40.598Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db243\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546213827
Jul 17 03:24:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:40.604Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx04\", asset_id=\"97\", container=\"dign-server-web\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Async\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546198318
Jul 17 03:24:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:40.963Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_labels\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"dashboard-metrics-scraper\", uid=\"ff15c352-595d-4ee7-9f33-949732c6163e\"}" timestamp=1689546220034
Jul 17 03:24:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:41.113Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546211647
Jul 17 03:24:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:41.186Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-sda4.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689546215764
Jul 17 03:24:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:41.195Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_MemHeapMaxM\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"JvmMetrics\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546219759
Jul 17 03:24:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:41.321Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_getCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"proxy_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"104e043c58ad6a4a5f519c65c2e6e973\", table=\"index_filename_proxy_hos_bucket\", type=\"index\"}" timestamp=1689546219759
Jul 17 03:24:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:41.418Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_filteredReadRequestCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", namespace=\"tsg_galaxy\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"7fed3599680b8cd8d453ece7a92c43c0\", table=\"gtpc_knowledge_base\", type=\"data\"}" timestamp=1689546219759
Jul 17 03:24:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:41.425Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"twa-tsgx04\", asset_id=\"97\", container=\"coredns\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"sleeping\"}" timestamp=1689546191603
Jul 17 03:24:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:41.444Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"java_lang_MemoryPool_UsageThreshold\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"Code Cache\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546219759
Jul 17 03:24:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:41.468Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_CheckAndPut_num_ops\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Server\"}" timestamp=1689546219759
Jul 17 03:24:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:41.657Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"jvm_memory_bytes_max\", area=\"heap\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546219759
Jul 17 03:24:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:41.773Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-tmpfiles-setup.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"service\"}" timestamp=1689546215764
Jul 17 03:24:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:41.874Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_system_seconds_total\", asset=\"twa-tsgx04\", asset_id=\"97\", container=\"monit-stream\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546198677
Jul 17 03:24:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:42.681Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Failed\", pod=\"local-path-provisioner-957fdf8bc-r2qbd\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"4ff67ed4-4465-4d8e-8bc9-8a9bac607a46\"}" timestamp=1689546215154
Jul 17 03:24:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:42.697Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_storeFileSize\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"proxy_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"104e043c58ad6a4a5f519c65c2e6e973\", table=\"index_filename_proxy_hos_bucket\", type=\"index\"}" timestamp=1689546219759
Jul 17 03:24:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:42.702Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", cpu=\"40\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Gohangout-msh-dellr740-dt002\", endpoint_id=\"167\", mode=\"idle\", module=\"DC-Gohangout\", module_id=\"35\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546222624
Jul 17 03:24:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:42.720Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_restart_policy\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-cm-policy-local-cache-7kss5\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", type=\"OnFailure\", uid=\"4d79d568-e597-4292-9848-bc8f41715e7a\"}" timestamp=1689546215154
Jul 17 03:24:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:42.734Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", cpu=\"7\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546216242
Jul 17 03:24:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:42.738Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", cpu=\"7\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546216242
Jul 17 03:24:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:42.757Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"cnibr0\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546216242
Jul 17 03:24:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:42.776Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_packets_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"vethbd01ab8d\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546216242
Jul 17 03:24:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:42.789Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db138\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546214838
Jul 17 03:24:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:42.981Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", cpu=\"33\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546216242
Jul 17 03:24:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:43.147Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", cpu=\"33\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546216242
Jul 17 03:24:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:43.170Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", cpu=\"44\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546216242
Jul 17 03:24:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:43.395Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_info\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_endpoint=\"cm-policy-local-cache-redis-metrics\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546215154
Jul 17 03:24:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:43.412Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_ports\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_endpoint=\"kubernetes-dashboard\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", port_name=\"http\", port_number=\"9090\", port_protocol=\"TCP\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546215154
Jul 17 03:24:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:43.471Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", container=\"metrics-server\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Total\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546217856
Jul 17 03:24:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:43.491Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_writes_merged_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda3\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546216092
Jul 17 03:24:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:43.571Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_created\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"cm-policy-local-cache-redis-master\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546215154
Jul 17 03:24:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:43.629Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_sector_writes_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda5\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546216092
Jul 17 03:24:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:43.725Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", cpu=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Gohangout-msh-dellr740-dt002\", endpoint_id=\"167\", module=\"DC-Gohangout\", module_id=\"35\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546222624
Jul 17 03:24:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:43.778Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_cache\", asset=\"pcap-tsgx01\", asset_id=\"71\", container=\"prometheus-alertmanager\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546219066
Jul 17 03:24:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:43.805Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_mspan_inuse_bytes\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"ipmi-exporter\"}" timestamp=1689546207619
Jul 17 03:24:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:49.158Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_incrementCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"firewall_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"9c59c1eb488a29ea5a3b4aee963a09b8\", table=\"index_filename_firewall_hos_bucket\", type=\"index\"}" timestamp=1689546227438
Jul 17 03:24:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:49.336Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_1_metric_tableWriteQueryPerSecond_5min_rate\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546227438
Jul 17 03:24:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:49.534Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_swap_limit_bytes\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"prometheus-server-configmap-reload\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546219050
Jul 17 03:24:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:49.585Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_annotations\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_endpoint=\"metrics-server\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546219668
Jul 17 03:24:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:49.623Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dnf-makecache.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"service\"}" timestamp=1689546221057
Jul 17 03:24:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:49.667Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_owner\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", owner_is_controller=\"true\", owner_kind=\"Deployment\", owner_name=\"dashboard-metrics-scraper\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"dashboard-metrics-scraper-5b94897dc9\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546219668
Jul 17 03:24:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:49.696Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx10\", asset_id=\"109\", cpu=\"30\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546221797
Jul 17 03:24:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:49.728Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_bytes_total\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens4f3np3\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546221797
Jul 17 03:24:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:49.783Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_writes_bytes_total\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"exporter\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546215188
Jul 17 03:24:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:49.901Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_session_record_hos_bucket_metric_minStoreFileAge\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546227438
Jul 17 03:24:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:49.954Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"getty-pre.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"target\"}" timestamp=1689546221057
Jul 17 03:24:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:49.986Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-rdma-devices-mlx5_6:1-umad.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689546221057
Jul 17 03:24:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:50.012Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"125\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546226709
Jul 17 03:24:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:50.045Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_gc_duration_seconds\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-olap003\", endpoint_id=\"235\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"yarn_resourcemanager,zookeeper,nacos,kafka,spark,druid_data,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\", quantile=\"0.75\"}" timestamp=1689546229714
Jul 17 03:24:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:50.051Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_user_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", container=\"prometheus-node-exporter\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546222903
Jul 17 03:24:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:50.070Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_193_1_metric_incrementTime_98th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546227438
Jul 17 03:24:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:50.086Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_compactionsFailedCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"proxy_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"4e3139889b447635cc39482c39741a58\", table=\"index_filename_proxy_hos_bucket\", type=\"index\"}" timestamp=1689546227438
Jul 17 03:24:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:50.107Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_limit_bytes\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"overlay_0-318\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546220457
Jul 17 03:24:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:50.161Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_mapped_file\", asset=\"twa-tsgx08\", asset_id=\"105\", container=\"firewall\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689546213991
Jul 17 03:24:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:50.231Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas_unavailable\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"traefik\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546221568
Jul 17 03:24:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:50.274Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_193_1_metric_compactionOutputSize_num_ops\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546227438
Jul 17 03:24:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:50.320Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_system_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", container=\"dign-client\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546224032
Jul 17 03:24:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:50.322Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_frame_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth5b68c36f\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546226709
Jul 17 03:24:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:50.328Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2dpartlabel-TSG\\\\x2dOS\\\\x2dSYSROOT.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689546226554
Jul 17 03:24:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:50.354Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_packets_dropped_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", interface=\"eth0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689546219423
Jul 17 03:24:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:50.356Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_latest_fork_seconds\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546227556
Jul 17 03:24:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:50.396Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_io_current\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"overlay_0-255\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546220457
Jul 17 03:24:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:50.400Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-ask-password-console.path\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"path\"}" timestamp=1689546226554
Jul 17 03:24:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:50.414Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_oom_events_total\", asset=\"twa-tsgx08\", asset_id=\"105\", container=\"telegraf\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689546209316
Jul 17 03:24:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:50.442Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_numReferenceFiles\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"proxy_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"a3b7319ec286102d93f51022aa81c973\", table=\"index_partfile_proxy_hos_bucket\", type=\"index\"}" timestamp=1689546227438
Jul 17 03:24:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:50.463Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_FlushMemstoreSize_75th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Server\"}" timestamp=1689546227438
Jul 17 03:24:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:50.492Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_created\", asset=\"twa-tsgx08\", asset_id=\"105\", configmap=\"kube-root-ca.crt\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546221568
Jul 17 03:24:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:50.495Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_packets_dropped_total\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", interface=\"eno34np1\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546216303
Jul 17 03:24:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:50.515Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"metrics-server\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"uninterruptible\"}" timestamp=1689546213339
Jul 17 03:24:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:50.525Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dracut-pre-mount.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"service\"}" timestamp=1689546226554
Jul 17 03:24:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:50.535Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_virtual_memory_bytes\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"chronyd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689546221057
Jul 17 03:24:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:50.535Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db65\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546227556
Jul 17 03:24:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:50.555Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"111\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546226709
Jul 17 03:24:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:50.573Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_193_1_metric_flushOutputSize_75th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546227438
Jul 17 03:24:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:50.582Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-6b43516fda13e768eb516cdc1c827c68d31eccf0f9e840e098fbeff3ac8f9cbc-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689546221057
Jul 17 03:24:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:50.687Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_multicast_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth6d7523c7\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546226709
Jul 17 03:24:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:50.709Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_packets_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth6d7523c7\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546226709
Jul 17 03:24:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:50.879Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx08\", asset_id=\"105\", container=\"dashboard-metrics-scraper\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Total\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546219134
Jul 17 03:24:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:50.891Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_alloc_bytes\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-olap003\", endpoint_id=\"235\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"yarn_resourcemanager,zookeeper,nacos,kafka,spark,druid_data,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546229714
Jul 17 03:24:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:50.896Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_system_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", container=\"kube-state-metrics\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546212664
Jul 17 03:24:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:50.940Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_inodes_free\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/e94a9495b59cd1a6e9531262f0bd6dc1693cc39a75c93bbdee486bc070e97f1c/shm\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546220457
Jul 17 03:24:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:50.958Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_cpu_shares\", asset=\"twa-tsgx08\", asset_id=\"105\", container=\"metrics\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546227561
Jul 17 03:24:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:51.021Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_swap_limit_bytes\", asset=\"twa-tsgx08\", asset_id=\"105\", container=\"exporter\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546227561
Jul 17 03:24:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:51.022Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx10\", asset_id=\"109\", cpu=\"29\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546221797
Jul 17 03:24:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:51.038Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_observed_generation\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"systemd-exporter\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546221568
Jul 17 03:24:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:51.124Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_annotations\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-master\", uid=\"aec1d9ea-4e0b-4dfd-a50c-48a3021e2496\"}" timestamp=1689546221568
Jul 17 03:24:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:51.141Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_io_current\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/data/var/lib/kubelet/pods/e4e2672a-a045-406b-b912-a701b409de63/volumes/kubernetes.io~projected/kube-api-access-qxdsd\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546220457
Jul 17 03:24:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:51.207Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_bytes_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens6f1np1\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546226709
Jul 17 03:24:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:51.233Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_info\", address=\"bc:97:e1:c6:a5:e0\", asset=\"twa-tsgx08\", asset_id=\"105\", broadcast=\"ff:ff:ff:ff:ff:ff\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"eno33np0\", duplex=\"full\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operstate=\"up\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546226709
Jul 17 03:24:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:51.246Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"27\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546226709
Jul 17 03:24:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:51.263Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-udev-trigger.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"oneshot\", type=\"service\"}" timestamp=1689546226554
Jul 17 03:24:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:51.288Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"abrt-vmcore.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"service\"}" timestamp=1689546226554
Jul 17 03:24:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:51.291Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_bytes_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", interface=\"ens3f1np1\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546225972
Jul 17 03:24:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:51.306Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"twa-tsgx08\", asset_id=\"105\", condition=\"Available\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"tsg-traffic-engine-numa-1-firewall\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"true\"}" timestamp=1689546221568
Jul 17 03:24:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:51.323Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_spec_type\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-master\", type=\"NodePort\", uid=\"aec1d9ea-4e0b-4dfd-a50c-48a3021e2496\"}" timestamp=1689546221568
Jul 17 03:24:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:51.472Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db166\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546227556
Jul 17 03:24:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:51.494Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_created\", asset=\"twa-tsgx08\", asset_id=\"105\", configmap=\"prometheus-server\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546221568
Jul 17 03:24:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:51.525Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_restart_policy\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"dign-server-zj2bv\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", type=\"Always\", uid=\"d4457315-9a37-45e8-b7b6-d04b15cea130\"}" timestamp=1689546221568
Jul 17 03:24:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:53.228Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"phy_rx_missed_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", dev=\"virtio_dign_c\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"mrzcpd-exporter\"}" timestamp=1689546228118
Jul 17 03:24:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:53.313Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_complete\", asset=\"msh-tsgx12\", asset_id=\"113\", condition=\"true\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", job_name=\"helm-delete-tsg-traffic-engine-numa-1\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546228663
Jul 17 03:24:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:53.341Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_size_bytes\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"shm\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/7bb3db6f1d82c5728eea04cb284934e26bebf61bb3dab8538198b0ff4346d2cf/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546223391
Jul 17 03:24:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:53.448Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-veth2169720f.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689546225496
Jul 17 03:24:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:53.459Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"msh-tsgx08\", asset_id=\"65\", cpu=\"62\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546228328
Jul 17 03:24:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:53.472Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"java_lang_Memory_Verbose\", asset=\"msh-dellr740-dt003\", asset_id=\"9\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt003-Hmaster\", endpoint_id=\"385\", module=\"DC-HBase\", module_id=\"3\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546232402
Jul 17 03:24:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:53.475Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_reads_merged_total\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"overlay_0-47\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546224182
Jul 17 03:24:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:53.479Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_cpu_seconds_total\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-burstable-pod4d27b507_e334_47c3_b9ec_ca1340a8a907.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"slice\"}" timestamp=1689546232135
Jul 17 03:24:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:53.495Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-4aec3d45710c2da2a877effba3feec670a88c2382a0d509ee111f58390efe287-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689546232135
Jul 17 03:24:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:53.498Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:20-0000:20:03.1-0000:21:00.1-infiniband-mlx5_3.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689546232135
Jul 17 03:24:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:53.521Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx08\", asset_id=\"65\", container=\"configmap-reloader\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Discard\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546230651
Jul 17 03:24:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:53.542Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-cd26913f\\\\x2dd130\\\\x2d4cb7\\\\x2da453\\\\x2d357d44e268db-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2d9zl28.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689546231179
Jul 17 03:24:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:53.589Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx08\", asset_id=\"65\", cpu=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546228328
Jul 17 03:24:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:53.625Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_up_changes_total\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens4f0np0\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546228328
Jul 17 03:24:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:53.632Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_bytes_total\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"cadvisor\", interface=\"ens7f1np1\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546219943
Jul 17 03:24:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:53.635Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_compressed_total\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens7f1np1\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546223391
Jul 17 03:24:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:53.657Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_compressed_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"vethfd1158fa\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546219409
Jul 17 03:24:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:53.669Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_scrape_collector_success\", asset=\"pcap-tsgx07\", asset_id=\"85\", collector=\"ipvs\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546219409
Jul 17 03:24:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:53.741Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_up_changes_total\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens3f1np1\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546228328
Jul 17 03:24:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:53.760Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_info\", address=\"00:00:00:00:00:00\", asset=\"pcap-tsgx07\", asset_id=\"85\", broadcast=\"00:00:00:00:00:00\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"lo\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operstate=\"unknown\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546219409
Jul 17 03:24:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:53.777Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_rss\", asset=\"msh-tsgx08\", asset_id=\"65\", container=\"prometheus-server-configmap-reload\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546221964
Jul 17 03:24:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:53.783Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_usage_bytes\", asset=\"pcap-tsgx07\", asset_id=\"85\", container=\"metrics\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546227687
Jul 17 03:24:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:53.828Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_info\", asset=\"msh-tsgx08\", asset_id=\"65\", container=\"firewall\", container_id=\"containerd://bb6378634d1234e08b1ec8ba963a7a411e71f14fd87caaf30e186810778b1ff9\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", image=\"registry.gdnt-cloud.website/tsg-firewall:v23.07-rc1-079e964\", image_id=\"sha256:2da56689d8dcb0909804cd0d6ba44926a8ef8c6579be490ff4cd41aeab743e73\", image_spec=\"registry.gdnt-cloud.website/tsg-firewall:v23.07-rc1-079e964\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", pod=\"tsg-traffic-engine-numa-0-firewall-57887f4b7c-6n8x9\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"b2a5a946-4882-4457-be8b-5aa14861ba28\"}" timestamp=1689546228950
Jul 17 03:24:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:53.829Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_packets_total\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"cadvisor\", interface=\"ens7f0np0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546219943
Jul 17 03:24:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:53.841Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx08\", asset_id=\"65\", cpu=\"28\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546228328
Jul 17 03:24:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:53.849Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"auditd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689546231179
Jul 17 03:24:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:53.858Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx08\", asset_id=\"65\", cpu=\"10\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546228328
Jul 17 03:24:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:53.859Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_terminated\", asset=\"msh-tsgx08\", asset_id=\"65\", container=\"helm\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-mrzcpd-exporter-4rjwg\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"9b1ef315-33f8-4a3d-a750-cb303d9afe8c\"}" timestamp=1689546228950
Jul 17 03:24:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:53.860Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"msh-tsgx12\", asset_id=\"113\", condition=\"Progressing\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"systemd-exporter\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"unknown\"}" timestamp=1689546228663
Jul 17 03:24:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:53.866Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-infiniband-umad6.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689546231179
Jul 17 03:24:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:53.875Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx08\", asset_id=\"65\", cpu=\"61\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546228328
Jul 17 03:24:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:53.876Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"msh-tsgx08\", asset_id=\"65\", condition=\"Progressing\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"configmap-reloader\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"unknown\"}" timestamp=1689546228950
Jul 17 03:24:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:53.896Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_persistentvolumeclaim_access_mode\", access_mode=\"ReadWriteOnce\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", persistentvolumeclaim=\"prometheus-server\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546228663
Jul 17 03:24:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:53.904Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_info\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"chart-values-ipmi-exporter\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546228950
Jul 17 03:24:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:53.942Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx08\", asset_id=\"65\", cpu=\"73\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546228328
Jul 17 03:24:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:53.958Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx08\", asset_id=\"65\", cpu=\"110\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546228328
Jul 17 03:24:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:53.987Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"var-lib-kubelet-pods-b2a5a946\\\\x2d4882\\\\x2d4457\\\\x2dbe8b\\\\x2d5aa14861ba28-volume\\\\x2dsubpaths-telegraf-telegraf-0.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689546232135
Jul 17 03:24:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:54.028Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_running\", asset=\"msh-tsgx12\", asset_id=\"113\", container=\"telegraf\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", pod=\"tsg-traffic-engine-numa-1-firewall-89d4cc897-2jc89\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"8ba148c1-f25f-4fa4-82b3-03b9d5a8ace6\"}" timestamp=1689546228663
Jul 17 03:24:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:54.044Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_discards_transmitted_total\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_8\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546223391
Jul 17 03:24:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:54.058Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"phy_tx_error_total\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", dev=\"virtio_dign_c\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"mrzcpd-exporter\"}" timestamp=1689546224559
Jul 17 03:24:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:54.064Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-67eafe01\\\\x2d721d\\\\x2d41b1\\\\x2d93f6\\\\x2d903b88527edb-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2d8lmkt.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689546231179
Jul 17 03:24:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:54.074Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_cpu_period\", asset=\"msh-tsgx08\", asset_id=\"65\", container=\"dign-server-web\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546232907
Jul 17 03:24:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:54.138Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_memory_Active_bytes\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546219409
Jul 17 03:24:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:54.157Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_packets_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"vethd26d000c\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546219409
Jul 17 03:24:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:54.203Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_down_changes_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth94114e67\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546229337
Jul 17 03:24:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:54.206Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Tcp_Bypass_STM\", app_name=\"sapp4\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", vsysId=\"1\"}" timestamp=1689546221765
Jul 17 03:24:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:54.227Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_namespace_created\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"consul\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546228950
Jul 17 03:24:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:54.243Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_metadata_resource_version\", asset=\"pcap-tsgx07\", asset_id=\"85\", configmap=\"certstore-tsg-traffic-engine-numa-1\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546222058
Jul 17 03:24:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:54.246Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_net_dev_group\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"bond0\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546229337
Jul 17 03:24:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:54.256Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_ports\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_endpoint=\"prometheus-pushgateway\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", port_name=\"http\", port_number=\"9091\", port_protocol=\"TCP\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546228950
Jul 17 03:24:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:54.276Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_reads_bytes_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", container=\"metrics-server\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546229471
Jul 17 03:24:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:54.281Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db138\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546219367
Jul 17 03:24:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:54.297Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", cpu=\"118\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546229337
Jul 17 03:24:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:54.301Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"clickhouse_os_irq_time_cpu8\", asset=\"pcap-dellr740-dt003\", asset_id=\"35\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"NC-Clickhouse-pcap-dellr740-dt003\", endpoint_id=\"85\", module=\"NC-Clickhouse\", module_id=\"29\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546233233
Jul 17 03:24:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:54.307Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", phase=\"Pending\", pod=\"dashboard-metrics-scraper-5b94897dc9-pbxbf\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"7af362d7-f44d-40a1-9857-36884a064be9\"}" timestamp=1689546224478
Jul 17 03:24:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:54.322Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_data_transmitted_bytes_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"mlx5_6\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546229337
Jul 17 03:24:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:54.355Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_completion_time\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-delete-tsg-traffic-engine-numa-0-7gg5s\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"6d5094f5-cbce-4a02-92f8-9c472f1ba305\"}" timestamp=1689546228663
Jul 17 03:24:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:54.373Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_info\", asset=\"msh-tsgx12\", asset_id=\"113\", created_by_kind=\"Job\", created_by_name=\"helm-install-prometheus\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", host_ip=\"10.10.20.172\", host_network=\"false\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", node=\"localhost\", nz_agent_id=\"3\", pod=\"helm-install-prometheus-k5mgl\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"75952f28-65f8-4ba3-afae-3c693a5ebfb7\"}" timestamp=1689546228663
Jul 17 03:24:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:54.422Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"msh-tsgx08\", asset_id=\"65\", cpu=\"21\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546228328
Jul 17 03:24:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:54.429Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_ulimits_soft\", asset=\"msh-tsgx08\", asset_id=\"65\", container=\"dign-client\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", ulimit=\"max_open_files\"}" timestamp=1689546232462
Jul 17 03:24:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:54.455Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_device_id\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens8f1np1\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546228328
Jul 17 03:24:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:54.464Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"machine_nvm_avg_power_budget_watts\", asset=\"msh-tsgx12\", asset_id=\"113\", boot_id=\"32e23b4f-f7f1-4bce-bec6-fc6466beee05\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"cadvisor\", machine_id=\"93360108d9c94db38802fcd6d26fb168\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", system_uuid=\"4c4c4544-0046-5410-8042-cac04f324733\"}" timestamp=1689546220589
Jul 17 03:24:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:54.479Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db39\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546225184
Jul 17 03:24:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:54.480Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_scrape_collector_duration_seconds\", asset=\"msh-tsgx08\", asset_id=\"65\", collector=\"cpufreq\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546228328
Jul 17 03:24:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:54.497Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx08\", asset_id=\"65\", cpu=\"66\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546228328
Jul 17 03:24:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:54.519Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_down_changes_total\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens1f0np0\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546228328
Jul 17 03:24:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:54.596Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"msh-tsgx08\", asset_id=\"65\", cpu=\"126\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546228328
Jul 17 03:24:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:54.621Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_xfs_block_mapping_extent_list_compares_total\", asset=\"pcap-dellr740-dt002\", asset_id=\"33\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"sdb1\", endpoint=\"NC-Gohangout-pcap-dellr740-dt002\", endpoint_id=\"173\", module=\"DC-Gohangout\", module_id=\"35\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546232675
Jul 17 03:24:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:54.825Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"msh-tsgx08\", asset_id=\"65\", cpu=\"36\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546228328
Jul 17 03:24:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:54.861Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_cpu_shares\", asset=\"msh-tsgx12\", asset_id=\"113\", container=\"kube-state-metrics\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546220589
Jul 17 03:24:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:54.875Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_annotations\", asset=\"msh-tsgx12\", asset_id=\"113\", configmap=\"chart-content-ipmi-exporter\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546228663
Jul 17 03:24:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:54.910Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_writes_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda5\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546230010
Jul 17 03:24:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:55.027Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-alertmanager-6df7f457b6-gzscl\", project=\"TSG-OS\", project_id=\"7\", reason=\"Shutdown\", service=\"prometheus-kube-state-metrics\", uid=\"c62cc0c7-b670-44ac-be9c-f9cc027d18ec\"}" timestamp=1689546222058
Jul 17 03:24:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:55.044Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cpu=\"27\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546219409
Jul 17 03:24:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:55.061Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"msh-tsgx12\", asset_id=\"113\", condition=\"true\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"coredns-6cb546fb97-4zrr7\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"5782b922-edbe-45a2-a171-c3d8e1c63480\"}" timestamp=1689546228663
Jul 17 03:24:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:56.479Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_swap\", asset=\"pcap-tsgx03\", asset_id=\"73\", container=\"firewall\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689546228081
Jul 17 03:24:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:56.494Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_observed_generation\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"prometheus-alertmanager\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546233389
Jul 17 03:24:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:56.518Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", container=\"redis\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Total\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546226154
Jul 17 03:24:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:56.726Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_reads_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", container=\"prometheus-node-exporter\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546220087
Jul 17 03:24:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:56.742Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"52\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546225883
Jul 17 03:24:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:56.745Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-mrzcpd-exporter-zvpqx\", project=\"TSG-OS\", project_id=\"7\", reason=\"Shutdown\", service=\"prometheus-kube-state-metrics\", uid=\"3a11e61a-e3d2-4781-8ccb-d7bacf3473e1\"}" timestamp=1689546233389
Jul 17 03:24:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:56.801Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_info\", asset=\"pcap-tsgx03\", asset_id=\"73\", configmap=\"chart-content-cm-policy-local-cache\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546233389
Jul 17 03:24:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:56.812Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_metadata_resource_version\", asset=\"pcap-tsgx03\", asset_id=\"73\", configmap=\"kube-root-ca.crt\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"consul\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546233389
Jul 17 03:24:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:56.875Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth537eab8c\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546225883
Jul 17 03:24:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:56.952Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_timex_loop_time_constant\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546225883
Jul 17 03:24:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:56.956Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546225883
Jul 17 03:24:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:56.978Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_exporter_last_scrape_error\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546226475
Jul 17 03:24:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:57.031Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_packets_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth958fa935\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546225883
Jul 17 03:24:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:57.044Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-38de76dcf7e6403582d78376482fb4803a5140985365b252f0bfb66b75632669.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"scope\"}" timestamp=1689546223372
Jul 17 03:24:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:57.066Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"gssproxy.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"service\"}" timestamp=1689546223372
Jul 17 03:24:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:57.321Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"iscsi-shutdown.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689546223372
Jul 17 03:24:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:57.390Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:20-0000:20:03.1-0000:21:00.1-infiniband_mad-umad3.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689546223372
Jul 17 03:24:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:57.394Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"iscsid.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"service\"}" timestamp=1689546223372
Jul 17 03:24:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:57.420Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_threads_max\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546231302
Jul 17 03:24:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:57.426Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-ens8f0np0.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689546223372
Jul 17 03:24:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:57.432Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_labels\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"coredns\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546233389
Jul 17 03:24:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:57.489Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db13\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546226475
Jul 17 03:24:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:57.645Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_limit_bytes\", asset=\"pcap-tsgx03\", asset_id=\"73\", container=\"dign-server-dns\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546231563
Jul 17 03:24:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:57.708Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_completion_time\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-systemd-exporter-pw27n\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"6e4c6271-0705-4680-9a24-73bceb8107af\"}" timestamp=1689546233389
Jul 17 03:24:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:57.744Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546225883
Jul 17 03:24:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:57.776Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_compressed_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens4f2np2\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546225883
Jul 17 03:24:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:57.790Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-rdma-devices-mlx5_7:1-umad.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689546223372
Jul 17 03:24:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:57.801Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"96\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546225883
Jul 17 03:24:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:57.815Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"101\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546225883
Jul 17 03:24:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:57.839Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_limit_bytes\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/run\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546229205
Jul 17 03:24:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:57.888Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_packets_dropped_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"cadvisor\", interface=\"ens1f1np1\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546229205
Jul 17 03:24:58 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:58.007Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", cpu=\"0\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"OLAP-Node-Exporter-pcap-dellr740-dt001\", endpoint_id=\"251\", mode=\"softirq\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546237557
Jul 17 03:24:58 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:58.042Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", cpu=\"4\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"OLAP-Node-Exporter-pcap-dellr740-dt001\", endpoint_id=\"251\", mode=\"system\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546237557
Jul 17 03:24:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:59.738Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"msh-tsgx01\", asset_id=\"51\", condition=\"Available\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"prometheus-pushgateway\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"unknown\"}" timestamp=1689546227425
Jul 17 03:24:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:59.751Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_annotations\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"kubernetes\", uid=\"be64c893-9ff9-451b-ae73-960bd855c284\"}" timestamp=1689546227425
Jul 17 03:24:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:59.911Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_mapped_file\", asset=\"msh-tsgx01\", asset_id=\"51\", container=\"prometheus-pushgateway\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546231955
Jul 17 03:24:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:24:59.981Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_processes\", asset=\"msh-tsgx01\", asset_id=\"51\", container=\"prometheus-server-configmap-reload\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546220268
Jul 17 03:25:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:00.036Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_link_downed_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_11\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546229083
Jul 17 03:25:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:00.037Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Unknown\", pod=\"local-path-provisioner-957fdf8bc-7gcxw\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"dc0b66cf-ac45-4d06-8dbb-3494485cf1d5\"}" timestamp=1689546227425
Jul 17 03:25:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:00.163Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_cpu_shares\", asset=\"msh-tsgx01\", asset_id=\"51\", container=\"firewall\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689546234593
Jul 17 03:25:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:00.196Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_limit_bytes\", asset=\"msh-tsgx01\", asset_id=\"51\", container=\"firewall\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689546234593
Jul 17 03:25:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:00.205Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx01\", asset_id=\"51\", container=\"kubernetes-dashboard\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Write\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546233020
Jul 17 03:25:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:00.220Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"49\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546229083
Jul 17 03:25:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:00.393Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx01\", asset_id=\"51\", container=\"coredns\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Discard\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546227476
Jul 17 03:25:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:00.627Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dracut-shutdown-onfailure.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"oneshot\", type=\"service\"}" timestamp=1689546224879
Jul 17 03:25:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:00.681Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_info\", asset=\"msh-tsgx01\", asset_id=\"51\", container=\"firewall\", container_id=\"containerd://4dbd7eb51e70749a083dad8d4909e543ee021e77044334497de7f2402622711c\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", image=\"registry.gdnt-cloud.website/tsg-firewall:rel_23.07-20230707123747-777982d\", image_id=\"sha256:b512a528b65b9c057c34741dc7d6006bb9670a57f95d092fba1f4f9be6edc50e\", image_spec=\"registry.gdnt-cloud.website/tsg-firewall:rel_23.07-20230707123747-777982d\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", pod=\"tsg-traffic-engine-numa-1-firewall-6c6f4bc877-xxxz9\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"b3efc35a-4844-47d9-82c9-7d335d0d4eaa\"}" timestamp=1689546227425
Jul 17 03:25:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:00.682Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_annotations\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"metrics-server\", uid=\"e112d503-72fa-46df-904e-aaeb86e3db1d\"}" timestamp=1689546227425
Jul 17 03:25:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:00.810Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_reservation_limit_bytes\", asset=\"msh-tsgx01\", asset_id=\"51\", container=\"prometheus-server\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546234593
Jul 17 03:25:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:00.862Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas_unavailable\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"mrzcpd-exporter\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546227425
Jul 17 03:25:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:00.874Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_read_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/run\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546228370
Jul 17 03:25:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:00.882Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Unknown\", pod=\"helm-install-traefik-pk7nl\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"90133098-1b52-4329-973e-ffb63ab8e8b1\"}" timestamp=1689546227425
Jul 17 03:25:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:00.902Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dracut-pre-pivot.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"oneshot\", type=\"service\"}" timestamp=1689546224879
Jul 17 03:25:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:01.019Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_metadata_resource_version\", asset=\"msh-tsgx01\", asset_id=\"51\", configmap=\"certstore-tsg-traffic-engine-numa-0\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546227425
Jul 17 03:25:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:01.061Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_storageclass_created\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", storageclass=\"local-path\"}" timestamp=1689546227425
Jul 17 03:25:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:01.081Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_compressed_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth9b710207\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546229083
Jul 17 03:25:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:01.102Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"mrzcpd_hugepages_setup.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"service\"}" timestamp=1689546224879
Jul 17 03:25:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:01.138Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-ens2f1np1.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689546224879
Jul 17 03:25:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:01.209Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"timers.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"target\"}" timestamp=1689546224879
Jul 17 03:25:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:02.523Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx02\", asset_id=\"63\", cpu=\"103\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546238318
Jul 17 03:25:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:02.557Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx02\", asset_id=\"63\", cpu=\"46\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546238318
Jul 17 03:25:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:02.607Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas_available\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"prometheus-kube-state-metrics\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546231412
Jul 17 03:25:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:02.694Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-udevd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689546228447
Jul 17 03:25:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:02.842Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_mallocs_total\", asset=\"pcap-dellr740-dt002\", asset_id=\"33\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"OLAP-Node-Exporter-pcap-dellr740-dt002\", endpoint_id=\"253\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546242497
Jul 17 03:25:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:02.934Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas_updated\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"prometheus-pushgateway\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546231412
Jul 17 03:25:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:02.959Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_annotations\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-1-firewall-76f6cf59ff\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546231412
Jul 17 03:25:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:03.019Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx02\", asset_id=\"63\", cpu=\"75\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546238318
Jul 17 03:25:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:03.049Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_info\", address=\"bc:97:e1:c6:a3:02\", asset=\"msh-tsgx02\", asset_id=\"63\", broadcast=\"ff:ff:ff:ff:ff:ff\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"eno35np2\", duplex=\"unknown\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operstate=\"down\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546238318
Jul 17 03:25:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:03.069Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_aof_last_rewrite_duration_sec\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546229269
Jul 17 03:25:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:03.161Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx02\", asset_id=\"63\", cpu=\"104\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546238318
Jul 17 03:25:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:03.199Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"msh-tsgx02\", asset_id=\"63\", container=\"kube-state-metrics\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"stopped\"}" timestamp=1689546231866
Jul 17 03:25:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:03.208Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"msh-tsgx02\", asset_id=\"63\", cpu=\"62\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546238318
Jul 17 03:25:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:03.230Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"msh-tsgx02\", asset_id=\"63\", condition=\"unknown\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-delete-tsg-traffic-engine-numa-1-ddm55\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"de0dc5f3-860f-415f-8f70-95817cf0299d\"}" timestamp=1689546231412
Jul 17 03:25:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:03.345Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_virtual_memory_max_bytes\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"k3s.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689546228447
Jul 17 03:25:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:03.366Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"82\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546238318
Jul 17 03:25:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:03.402Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_user_seconds_total\", asset=\"msh-tsgx02\", asset_id=\"63\", container=\"lb-tcp-443\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546218890
Jul 17 03:25:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:03.411Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_netstat_Udp_InDatagrams\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546238318
Jul 17 03:25:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:03.428Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_info\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_endpoint=\"cm-policy-local-cache-redis-master\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546231412
Jul 17 03:25:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:03.432Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db2\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546229269
Jul 17 03:25:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:03.451Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx02\", asset_id=\"63\", cpu=\"61\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546238318
Jul 17 03:25:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:03.454Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_status_active\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", job_name=\"helm-install-mrzcpd-exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546231412
Jul 17 03:25:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:03.502Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_fifo_total\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"eno1\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546238318
Jul 17 03:25:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:03.516Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx02\", asset_id=\"63\", cpu=\"64\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546238318
Jul 17 03:25:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:03.549Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"net_conntrack_dialer_conn_failed_total\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", dialer_name=\"181\", instance=\"10.10.20.159:10090\", reason=\"resolution\", type=\"global\"}" timestamp=1689546242990
Jul 17 03:25:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:03.568Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_scaling_frequency_min_hertz\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", cpu=\"44\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-dt001\", endpoint_id=\"225\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546242988
Jul 17 03:25:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:03.689Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"net_conntrack_dialer_conn_failed_total\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", dialer_name=\"183\", instance=\"10.10.20.159:10090\", reason=\"resolution\", type=\"global\"}" timestamp=1689546242990
Jul 17 03:25:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:03.716Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db216\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546229269
Jul 17 03:25:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:03.728Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_queue_length\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"cnibr0\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546238318
Jul 17 03:25:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:03.802Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_edac_csrow_uncorrectable_errors_total\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", controller=\"3\", csrow=\"unknown\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-dt001\", endpoint_id=\"225\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546242988
Jul 17 03:25:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:03.849Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_xfs_extent_allocation_blocks_allocated_total\", asset=\"pcap-dellr740-dt002\", asset_id=\"33\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"sdb1\", endpoint=\"OLAP-Node-Exporter-pcap-dellr740-dt002\", endpoint_id=\"253\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546242497
Jul 17 03:25:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:03.894Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-19e626095d3782c6fc90a46b8fca2ae0d75e7c10299930eba0a129115923ef3a-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689546228447
Jul 17 03:25:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:03.906Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_load_average_10s\", asset=\"msh-tsgx02\", asset_id=\"63\", container=\"telegraf-security\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689546223789
Jul 17 03:25:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:04.005Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"prometheus_target_scrape_pool_sync_total\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", instance=\"10.10.20.159:10090\", scrape_job=\"399\", type=\"global\"}" timestamp=1689546242990
Jul 17 03:25:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:04.022Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_frequency_min_hertz\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", cpu=\"8\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-dt001\", endpoint_id=\"225\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546242988
Jul 17 03:25:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:04.095Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx02\", asset_id=\"63\", cpu=\"66\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546238318
Jul 17 03:25:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:04.121Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", state=\"sleeping\"}" timestamp=1689546221914
Jul 17 03:25:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:04.139Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_annotations\", asset=\"msh-tsgx02\", asset_id=\"63\", configmap=\"sce-tsg-traffic-engine-numa-0\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546231412
Jul 17 03:25:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:04.768Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_xfs_block_mapping_reads_total\", asset=\"msh-dellr740-olap005\", asset_id=\"19\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"sda4\", endpoint=\"NC-Spark-msh-dellr740-olap005\", endpoint_id=\"17\", module=\"NC-Spark\", module_id=\"47\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546243391
Jul 17 03:25:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:06.489Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", cpu=\"33\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Spark-msh-dellr740-olap003\", endpoint_id=\"13\", mode=\"user\", module=\"NC-Spark\", module_id=\"47\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546246092
Jul 17 03:25:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:06.778Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-ens3f1np1.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689546233582
Jul 17 03:25:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:06.785Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_compressed_total\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"eno2\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546243910
Jul 17 03:25:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:06.848Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_cpu_seconds_total\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-poda99c67a0_fa8b_483f_afda_9724525367bb.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"slice\"}" timestamp=1689546233582
Jul 17 03:25:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:06.863Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_max_fds\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"udisks2.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689546233582
Jul 17 03:25:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:06.895Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_info\", asset=\"twa-tsgx01\", asset_id=\"91\", created_by_kind=\"Job\", created_by_name=\"helm-install-ipmi-exporter\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", host_ip=\"10.10.30.161\", host_network=\"false\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", node=\"localhost\", nz_agent_id=\"3\", pod=\"helm-install-ipmi-exporter-n5l8f\", pod_ip=\"10.42.0.157\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"b0fbdc3c-7bf3-43b8-8574-66286b27a6a0\"}" timestamp=1689546241227
Jul 17 03:25:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:06.988Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_running\", asset=\"twa-tsgx01\", asset_id=\"91\", container=\"exporter-merger\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"mrzcpd-exporter-5cf7f967cc-97flx\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"083e9092-7400-4cad-afc0-cffbf05464a4\"}" timestamp=1689546241227
Jul 17 03:25:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:06.995Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-34ba848e05d4ba2a53edc9114a2a216d54aa396606f7b4fb7d113518104d40b5-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689546233582
Jul 17 03:25:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:07.000Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"twa-tsgx01\", asset_id=\"91\", condition=\"false\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-traefik-7djww\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"49249a6d-7cf8-40c5-91f7-df8d1684f04c\"}" timestamp=1689546241227
Jul 17 03:25:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:07.021Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx01\", asset_id=\"91\", container=\"firewall\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Async\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689546237880
Jul 17 03:25:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:07.025Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-37167b92\\\\x2dd06b\\\\x2d4d10\\\\x2d8d55\\\\x2d65300f19e016-volumes-kubernetes.io\\\\x7esecret-kubernetes\\\\x2ddashboard\\\\x2dcerts.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689546233582
Jul 17 03:25:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:07.063Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_annotations\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-traefik-7djww\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"49249a6d-7cf8-40c5-91f7-df8d1684f04c\"}" timestamp=1689546241227
Jul 17 03:25:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:07.212Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cpu=\"63\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546243910
Jul 17 03:25:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:07.227Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_compressed_total\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"eno34np1\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546243910
Jul 17 03:25:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:07.270Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Pkt_Outbound\", app_name=\"sapp4\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", vsysId=\"1\"}" timestamp=1689546244123
Jul 17 03:25:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:07.347Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cryptsetup.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"target\"}" timestamp=1689546233582
Jul 17 03:25:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:07.534Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_created\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.tsg-traffic-engine-numa-0.v2\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546241227
Jul 17 03:25:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:07.642Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"scrape_samples_scraped\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_instance=\"10.42.0.173:9121\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546238290
Jul 17 03:25:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:07.660Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_owner\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", owner_is_controller=\"true\", owner_kind=\"Deployment\", owner_name=\"tsg-traffic-engine-numa-1-firewall\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-1-firewall-5cb6f857dc\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546241227
Jul 17 03:25:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:07.912Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cpu=\"10\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546243910
Jul 17 03:25:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:07.935Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cpu=\"53\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546243910
Jul 17 03:25:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:08.012Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_device_id\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"lo\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546243910
Jul 17 03:25:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:08.093Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_status_completion_time\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", job_name=\"helm-install-traefik-crd\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546241227
Jul 17 03:25:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:08.204Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-45cfa87f\\\\x2d48cf\\\\x2d44fc\\\\x2da587\\\\x2d24a36ba4ecdb-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2dg2w2t.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689546233582
Jul 17 03:25:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:08.288Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cpu=\"90\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546243910
Jul 17 03:25:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:08.336Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sshd-keygen@rsa.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"oneshot\", type=\"service\"}" timestamp=1689546233582
Jul 17 03:25:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:11.370Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_1_metric_deleteTime_mean\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546250148
Jul 17 03:25:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:11.388Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_compactionsCompletedCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"a333e50da1147ae77e67987dafdfcedd\", table=\"rtp_hos_bucket\", type=\"data\"}" timestamp=1689546250148
Jul 17 03:25:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:12.403Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_194_1_metric_majorCompactionOutputFileCount_mean\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546250148
Jul 17 03:25:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:12.676Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_storeFileSize\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"firewall_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"de3e9af7463fe090b6840184e6339540\", table=\"index_time_firewall_hos_bucket\", type=\"index\"}" timestamp=1689546250148
Jul 17 03:25:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:12.728Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_194_3_metric_putBatchTime_99_9th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546250148
Jul 17 03:25:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:12.767Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_deleteCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"rtp_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"afda67220314ac681ee8f19c890bcc36\", table=\"index_partfile_rtp_hos_bucket\", type=\"index\"}" timestamp=1689546250148
Jul 17 03:25:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:12.846Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_1_metric_majorCompactionOutputFileCount_95th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546250148
Jul 17 03:25:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:12.865Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_3_metric_majorCompactionOutputSize_95th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546250148
Jul 17 03:25:13 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:13.579Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_tsg_table_report_result_metric_majorCompactionTime_95th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546250148
Jul 17 03:25:13 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:13.940Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_tsg_table_report_result_metric_deleteTime_num_ops\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546250148
Jul 17 03:25:13 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:13.976Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_MemHeapMaxM\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"JvmMetrics\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546250148
Jul 17 03:25:14 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:14.005Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_numBytesCompactedCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"default\", project=\"OLAP\", project_id=\"3\", region=\"42f47eace43d0ea637cdff9f25c5307f\", table=\"index_time_default\", type=\"index\"}" timestamp=1689546250148
Jul 17 03:25:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:15.269Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_annotations\", asset=\"twa-tsgx07\", asset_id=\"103\", configmap=\"cm-policy-local-cache-redis-scripts\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546248076
Jul 17 03:25:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:15.488Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_info\", asset=\"twa-tsgx07\", asset_id=\"103\", configmap=\"sapp-tsg-traffic-engine-numa-0\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546248076
Jul 17 03:25:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:15.613Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link_mode\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth88a71bbf\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546241230
Jul 17 03:25:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:16.238Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"50\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546241230
Jul 17 03:25:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:16.438Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-pod18136b1d_dee5_4f2e_a352_07c6fe3b2565.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"slice\"}" timestamp=1689546247046
Jul 17 03:25:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:16.785Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"46\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546241230
Jul 17 03:25:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:16.949Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_labels\", asset=\"twa-tsgx07\", asset_id=\"103\", configmap=\"shaping-tsg-traffic-engine-numa-0\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546248076
Jul 17 03:25:17 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:17.138Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"44\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546241230
Jul 17 03:25:17 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:17.166Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_protocol_type\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethe2047d8e\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546241230
Jul 17 03:25:17 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:17.209Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cpupower.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"service\"}" timestamp=1689546247046
Jul 17 03:25:17 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:17.558Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"52\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546241230
Jul 17 03:25:17 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:17.585Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_created\", asset=\"twa-tsgx07\", asset_id=\"103\", configmap=\"kube-root-ca.crt\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546248076
Jul 17 03:25:17 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:17.661Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_usage_bytes\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689546246132
Jul 17 03:25:17 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:17.677Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_info\", asset=\"twa-tsgx07\", asset_id=\"103\", container=\"firewall\", container_id=\"containerd://d5fdb9f7a199cdf3bbfc06b2d761364c7bcda5d0af25b97ff2276bb33e7b8ece\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", image=\"registry.gdnt-cloud.website/tsg-firewall:v23.07-rc1-079e964\", image_id=\"sha256:2da56689d8dcb0909804cd0d6ba44926a8ef8c6579be490ff4cd41aeab743e73\", image_spec=\"registry.gdnt-cloud.website/tsg-firewall:v23.07-rc1-079e964\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", pod=\"tsg-traffic-engine-numa-0-firewall-5b47889f4f-8qqtj\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"64082a69-3be2-4b5e-9e3a-c0da16576d9e\"}" timestamp=1689546248076
Jul 17 03:25:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:20.039Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_spec_type\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"traefik\", type=\"LoadBalancer\", uid=\"2b43699a-fd09-42be-ab6f-ab11003e92f0\"}" timestamp=1689546253949
Jul 17 03:25:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:20.176Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"cadvisor\", failure_type=\"pgmajfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"container\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689546249984
Jul 17 03:25:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:20.198Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_namespace_status_phase\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", phase=\"Active\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546253949
Jul 17 03:25:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:20.340Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_usage_bytes\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689546250038
Jul 17 03:25:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:20.360Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"7\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546248263
Jul 17 03:25:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:20.373Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sockets.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"target\"}" timestamp=1689546250657
Jul 17 03:25:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:20.380Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-bc823522\\\\x2d8a2a\\\\x2d46b8\\\\x2d99b2\\\\x2d927c278ba36b-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2dnxcqx.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689546250657
Jul 17 03:25:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:20.600Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-infiniband-umad8.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689546250657
Jul 17 03:25:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:20.670Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dracut-pre-mount.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"service\"}" timestamp=1689546250657
Jul 17 03:25:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:20.673Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546257612
Jul 17 03:25:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:20.758Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failcnt\", asset=\"twa-tsgx06\", asset_id=\"101\", container=\"traefik\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546242865
Jul 17 03:25:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:20.768Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_compressed_total\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens6f1np1\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546257612
Jul 17 03:25:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:20.901Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_restarts_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", container=\"telegraf\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", pod=\"tsg-traffic-engine-numa-1-firewall-5d7bfbbbfb-4wgxr\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"ab53e5a7-f22e-498d-b5af-535ae7359350\"}" timestamp=1689546253828
Jul 17 03:25:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:20.928Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_status_completion_time\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", job_name=\"helm-install-traefik\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546253828
Jul 17 03:25:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:20.992Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_virtual_memory_bytes\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"watchdog.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689546250657
Jul 17 03:25:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:21.045Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2dpath-pci\\\\x2d0000:81:00.0\\\\x2dscsi\\\\x2d0:2:0:0\\\\x2dpart3.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689546244356
Jul 17 03:25:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:21.088Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"31\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546257612
Jul 17 03:25:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:21.165Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"64\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546257612
Jul 17 03:25:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:21.184Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"60\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546257612
Jul 17 03:25:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:21.199Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"shm\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/4cb5f0e8ce74a18b62361d9117cccdb52384a274068f68042f305e7447d47135/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546248263
Jul 17 03:25:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:21.223Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"67\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546257612
Jul 17 03:25:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:21.553Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db140\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546254840
Jul 17 03:25:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:21.560Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"abrt-xorg.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"simple\", type=\"service\"}" timestamp=1689546250657
Jul 17 03:25:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:21.575Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.grpc.v1.cri-sandboxes-35d12a8edd12430a26da1af1fe9aafbafd3a0645ba33ecb391cf8e2aa585b6a7-shm.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689546250657
Jul 17 03:25:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:21.602Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_type\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"chart-values-ipmi-exporter\", service=\"prometheus-kube-state-metrics\", type=\"Opaque\"}" timestamp=1689546253828
Jul 17 03:25:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:21.603Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"initrd-udevadm-cleanup-db.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689546250657
Jul 17 03:25:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:21.616Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_limit_bytes\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda4\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546252223
Jul 17 03:25:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:21.629Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", container=\"kube-state-metrics\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Read\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546243216
Jul 17 03:25:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:21.656Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", phase=\"Unknown\", pod=\"prometheus-alertmanager-6df7f457b6-z42km\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"26d0147a-fc67-4990-96ac-7d6c8e980d26\"}" timestamp=1689546253828
Jul 17 03:25:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:21.736Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_metadata_resource_version\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.tsg-traffic-engine-numa-1.v4\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546253828
Jul 17 03:25:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:22.589Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_xfs_block_mapping_unmaps_total\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"sda4\", endpoint=\"OLAP-Node-Exporter-twa-dellr740-dt001\", endpoint_id=\"257\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546261405
Jul 17 03:25:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:23.507Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_info\", asset=\"msh-tsgx09\", asset_id=\"55\", cluster_ip=\"10.43.229.121\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", uid=\"47622ec4-540e-48f7-b6bb-36ed2f8c621d\"}" timestamp=1689546260068
Jul 17 03:25:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:23.535Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_edac_csrow_correctable_errors_total\", asset=\"msh-tsgx09\", asset_id=\"55\", controller=\"1\", csrow=\"unknown\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546254790
Jul 17 03:25:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:23.559Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_compressed_total\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth9328e3a5\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546254790
Jul 17 03:25:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:23.592Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"msh-tsgx09\", asset_id=\"55\", condition=\"unknown\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-kube-state-metrics-6b768c7bf8-db9cd\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"1c96629c-a0c6-4e4c-8e97-b6bbe91aa6d9\"}" timestamp=1689546260068
Jul 17 03:25:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:23.758Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_memory_AnonHugePages_bytes\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546254790
Jul 17 03:25:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:23.802Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_errs_total\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens4f0np0\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546254790
Jul 17 03:25:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:23.984Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"msh-tsgx09\", asset_id=\"55\", condition=\"unknown\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-server-6ff784d758-cs9wf\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"58676ba8-2676-43fd-b12b-038d471289b5\"}" timestamp=1689546260068
Jul 17 03:25:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:24.128Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"msh-dellr740-olap004\", asset_id=\"17\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Spark-msh-dellr740-olap004\", endpoint_id=\"15\", module=\"NC-Spark\", module_id=\"47\", name=\"44\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", type=\"Processor\"}" timestamp=1689546262253
Jul 17 03:25:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:24.263Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546254790
Jul 17 03:25:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:24.348Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Udp_Link_S2C_ALL\", app_name=\"sapp4\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", vsysId=\"1\"}" timestamp=1689546260446
Jul 17 03:25:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:24.579Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_info\", address=\"bc:97:e1:c6:88:f1\", asset=\"msh-tsgx09\", asset_id=\"55\", broadcast=\"ff:ff:ff:ff:ff:ff\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"eno34np1\", duplex=\"unknown\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operstate=\"down\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546254790
Jul 17 03:25:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:24.593Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_memory_used_peak_bytes\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546257019
Jul 17 03:25:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:24.652Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"rx_pkts_total\", app=\"sapp4-0\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"nf_0_fw\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"mrzcpd-exporter\", serviceFunctionIndex=\"0\"}" timestamp=1689546260889
Jul 17 03:25:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:24.779Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"14\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546260160
Jul 17 03:25:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:24.797Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_address_assign_type\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"virtio_dign_s\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546260160
Jul 17 03:25:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:25.029Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"75\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546260160
Jul 17 03:25:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:25.056Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-930d15af2d3b24fe92be345ef51777909a1badb6f81b9919b3ad0b10b2f7d1b2.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"scope\"}" timestamp=1689546251102
Jul 17 03:25:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:25.091Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"prometheus-server-configmap-reload\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Sync\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546256247
Jul 17 03:25:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:25.100Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_address_assign_type\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"eno36np3\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546260160
Jul 17 03:25:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:25.145Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Pending\", pod=\"kube-multus-ds-dsd65\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"e9990f8b-7450-47a8-867f-7387f0ac7e25\"}" timestamp=1689546251122
Jul 17 03:25:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:25.179Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_down_changes_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"eno36np3\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546260160
Jul 17 03:25:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:25.191Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db221\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546261795
Jul 17 03:25:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:25.209Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"pcap-tsgx12\", asset_id=\"117\", condition=\"Available\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"dashboard-metrics-scraper\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"true\"}" timestamp=1689546251122
Jul 17 03:25:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:25.215Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Tcp_Link_S2C_ALL\", app_name=\"sapp4\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", vsysId=\"1\"}" timestamp=1689546252900
Jul 17 03:25:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:25.233Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"0\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546260160
Jul 17 03:25:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:25.248Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_speed_bytes\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"vethc96a0a69\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546260160
Jul 17 03:25:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:25.332Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"34\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546260160
Jul 17 03:25:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:25.350Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"lb-tcp-443\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"running\"}" timestamp=1689546259058
Jul 17 03:25:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:25.356Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"43\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546260160
Jul 17 03:25:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:26.414Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-virtio_dign_s.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689546253907
Jul 17 03:25:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:26.607Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_hwmon_chip_names\", asset=\"twa-tsgx09\", asset_id=\"107\", chip=\"0000:60:03_1_0000:63:00_2\", chip_name=\"bnxt_en\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546255810
Jul 17 03:25:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:26.658Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"vmtoolsd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689546253907
Jul 17 03:25:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:26.685Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-afefe92ea84fa41dbbadb1687ce5c63d370abc6861879fe40c18cca568cc3018.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"scope\"}" timestamp=1689546253907
Jul 17 03:25:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:27.105Z caller=dedupe.go:112 component=remote level=error remote_name=52db6f url=http://10.10.10.159:10090/nz-agent/prometheus/proxy/api/v1/write msg="non-recoverable error" count=1000 exemplarCount=0 err="server returned HTTP status 400 : out of order sample"
Jul 17 03:25:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:27.121Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_file_descriptors\", asset=\"twa-tsgx09\", asset_id=\"107\", container=\"lb-tcp-80\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546260852
Jul 17 03:25:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:27.349Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_metadata_resource_version\", asset=\"pcap-tsgx04\", asset_id=\"79\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx04\", endpoint_id=\"473\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"1\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.tsg-traffic-engine-numa-1.v3\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546263985
Jul 17 03:25:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:27.418Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx09\", asset_id=\"107\", container=\"dign-client\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Write\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546252922
Jul 17 03:25:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:27.435Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_protocol_type\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth16c89716\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546255810
Jul 17 03:25:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:27.455Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_commands_duration_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cmd=\"zremrangebyscore\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546263549
Jul 17 03:25:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:27.473Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_state_started\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"lb-tcp-80\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"svclb-traefik-dbdae63f-svgf7\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"b7d50d7a-bdb1-4746-8ca3-4cae9f1a4c87\"}" timestamp=1689546255682
Jul 17 03:25:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:27.485Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_cpu_shares\", asset=\"twa-tsgx09\", asset_id=\"107\", container=\"prometheus-server\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546265490
Jul 17 03:25:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:27.505Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_ulimits_soft\", asset=\"twa-tsgx09\", asset_id=\"107\", container=\"kubernetes-dashboard\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", ulimit=\"max_open_files\"}" timestamp=1689546254655
Jul 17 03:25:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:27.517Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_multicast_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"eno2\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546262882
Jul 17 03:25:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:27.532Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_files_free\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"shm\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/afefe92ea84fa41dbbadb1687ce5c63d370abc6861879fe40c18cca568cc3018/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546255810
Jul 17 03:25:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:27.559Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_memory_DirectMap1G_bytes\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546255810
Jul 17 03:25:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:27.644Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db241\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546259446
Jul 17 03:25:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:27.721Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_changes_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth30e28f20\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546262882
Jul 17 03:25:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:27.736Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"scrape_samples_post_metric_relabeling\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_instance=\"10.42.0.163:9290\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"ipmi-exporter\"}" timestamp=1689546261323
Jul 17 03:25:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:27.743Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_drop_total\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens3f1np1\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546255810
Jul 17 03:25:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:27.765Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_scrape_collector_success\", asset=\"twa-tsgx09\", asset_id=\"107\", collector=\"pressure\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546255810
Jul 17 03:25:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:28.123Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_oom_events_total\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"lb-tcp-443\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546258147
Jul 17 03:25:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:28.136Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-848571bc\\\\x2da31e\\\\x2d44de\\\\x2da0f0\\\\x2dd172cf8f3bd0-volume\\\\x2dsubpaths-config\\\\x2dvolume-telegraf\\\\x2dsecurity-2.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689546254114
Jul 17 03:25:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:28.144Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_device_id\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth68516c90\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546262882
Jul 17 03:25:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:28.158Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_drop_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens1f1np1\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546262882
Jul 17 03:25:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:28.191Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"56\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546262882
Jul 17 03:25:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:28.208Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"91\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546262882
Jul 17 03:25:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:28.212Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_replication_backlog_bytes\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546259446
Jul 17 03:25:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:28.223Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"promhttp_metric_handler_errors_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cause=\"encoding\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546262882
Jul 17 03:25:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:28.265Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_colls_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens8f0np0\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546262882
Jul 17 03:25:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:28.281Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"msh-tsgx10\", asset_id=\"69\", condition=\"unknown\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"coredns-647dbfccd5-sr892\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"ce676e45-7527-495c-945c-c9fa2e1248cd\"}" timestamp=1689546255682
Jul 17 03:25:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:28.358Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_net_dev_group\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth6f518146\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546262882
Jul 17 03:25:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:28.433Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-virtual-net-veth476a8450.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689546254114
Jul 17 03:25:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:28.448Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-ens4f3np3.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689546254114
Jul 17 03:25:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:28.467Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"28\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546262882
Jul 17 03:25:30 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:30.972Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_fully_labeled_replicas\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-1-firewall-7fd8897697\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546264068
Jul 17 03:25:30 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:30.984Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"118\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546260939
Jul 17 03:25:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:31.047Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_daemonset_annotations\", asset=\"msh-tsgx05\", asset_id=\"67\", daemonset=\"kube-multus-ds\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546264068
Jul 17 03:25:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:31.062Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_waiting\", asset=\"msh-tsgx05\", asset_id=\"67\", container=\"telegraf\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", pod=\"tsg-traffic-engine-numa-1-firewall-6d869d4b4c-7pv8f\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"d8012c00-aaec-4f56-a35b-ad6f4d1ef590\"}" timestamp=1689546264068
Jul 17 03:25:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:31.430Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"57\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546260939
Jul 17 03:25:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:31.465Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"35\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546260939
Jul 17 03:25:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:31.466Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_alloc_bytes\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546260939
Jul 17 03:25:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:31.491Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_bytes_total\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens1f1np1\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546260939
Jul 17 03:25:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:31.495Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_status_start_time\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", job_name=\"helm-install-tsg-traffic-engine-numa-1\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546264068
Jul 17 03:25:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:31.506Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_annotations\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"chart-values-traefik-crd\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546264068
Jul 17 03:25:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:31.528Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_dormant\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"virtio_dign_c\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546260939
Jul 17 03:25:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:31.544Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"34\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546260939
Jul 17 03:25:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:31.553Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db253\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546270141
Jul 17 03:25:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:31.669Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"119\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546260939
Jul 17 03:25:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:31.688Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"39\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546260939
Jul 17 03:25:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:31.713Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"20\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546260791
Jul 17 03:25:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:31.730Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Send_Tcp_S_A\", app_name=\"sapp4\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", vsysId=\"1\"}" timestamp=1689546269611
Jul 17 03:25:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:31.960Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_packets_total\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens6f0np0\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546259093
Jul 17 03:25:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:32.001Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db179\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546266438
Jul 17 03:25:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:32.064Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"twa-tsgx03\", asset_id=\"95\", condition=\"unknown\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-node-exporter-dd5xk\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"18031228-abd9-478c-bf5c-0acdbff9f2e6\"}" timestamp=1689546270436
Jul 17 03:25:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:32.118Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"71\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546260939
Jul 17 03:25:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:32.188Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_persistentvolume_labels\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", persistentvolume=\"pvc-0f67f9a1-1515-49ba-bfe0-9e91f648a16f\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546270436
Jul 17 03:25:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:32.350Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-6e410083\\\\x2d6316\\\\x2d4234\\\\x2d8b1e\\\\x2d7c79ee4588d4-volume\\\\x2dsubpaths-config\\\\x2dpath-dign\\\\x2dclient-3.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689546266907
Jul 17 03:25:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:32.753Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"msh-tsgx11\", asset_id=\"111\", condition=\"false\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-cm-policy-local-cache-k4zkb\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"9773a9ea-4c23-4577-b88b-c11f7277b019\"}" timestamp=1689546259276
Jul 17 03:25:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:32.781Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_created\", asset=\"twa-tsgx03\", asset_id=\"95\", configmap=\"chart-content-traefik-crd\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546270436
Jul 17 03:25:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:32.790Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_info\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_endpoint=\"prometheus-alertmanager\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546270436
Jul 17 03:25:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:32.811Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"slice\"}" timestamp=1689546266907
Jul 17 03:25:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:32.814Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"114\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546259093
Jul 17 03:25:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:32.833Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546259093
Jul 17 03:25:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:32.940Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_state_started\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"telegraf-security\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", pod=\"tsg-traffic-engine-numa-0-firewall-99cddfc69-25wpt\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"abf27d03-a067-448d-81a8-11df769a2208\"}" timestamp=1689546259276
Jul 17 03:25:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:32.949Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546256711
Jul 17 03:25:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:32.949Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_state_id\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_9\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546259093
Jul 17 03:25:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:33.067Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"jvm_memory_bytes_max\", area=\"nonheap\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-Yarn-twa-dellr740-dt001-NodeManager\", endpoint_id=\"363\", module=\"DC-Yarn\", module_id=\"1\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546271354
Jul 17 03:25:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:33.184Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_mapped_file\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689546262280
Jul 17 03:25:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:33.316Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_multicast_total\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens3f1np1\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546259093
Jul 17 03:25:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:33.326Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"rpcbind.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689546262644
Jul 17 03:25:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:33.338Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_fifo_total\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth914fec83\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546259093
Jul 17 03:25:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:33.431Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Pkt_Inbound\", app_name=\"sapp4\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", vsysId=\"1\"}" timestamp=1689546260718
Jul 17 03:25:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:33.549Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_down_changes_total\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens3f0np0\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546260605
Jul 17 03:25:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:33.596Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cpupower.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"service\"}" timestamp=1689546267691
Jul 17 03:25:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:33.614Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_id\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethc3a959fd\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546260605
Jul 17 03:25:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:33.676Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-pushgateway-bd476bcf9-5mpjx\", project=\"TSG-OS\", project_id=\"7\", reason=\"NodeAffinity\", service=\"prometheus-kube-state-metrics\", uid=\"f2075fac-2ce7-4f3c-b94c-4c05e2de4805\"}" timestamp=1689546259894
Jul 17 03:25:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:33.679Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"113\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546260605
Jul 17 03:25:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:33.705Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-62c6126e02a7e8ff8e71d68bd4baa2dd72c888dbf02a1f8962d370e89479191a.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"scope\"}" timestamp=1689546267691
Jul 17 03:25:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:33.720Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-podf2034e38_7d8b_4f82_8274_cb5f14eacda8.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"slice\"}" timestamp=1689546267691
Jul 17 03:25:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:33.764Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_load_average_10s\", asset=\"twa-tsgx05\", asset_id=\"99\", container=\"telegraf\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689546243883
Jul 17 03:25:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:33.800Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-34fd0e1b399c88f828180af36a9887dc42a1f82a23c464e843f545c11049bcd9.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"scope\"}" timestamp=1689546267691
Jul 17 03:25:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:33.910Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"39\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546260605
Jul 17 03:25:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:33.930Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-37526140\\\\x2d01d6\\\\x2d46f5\\\\x2d9f0b\\\\x2de066a6254bac-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2dcftdn.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689546267691
Jul 17 03:25:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:33.932Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_readonly\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"shm\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/f113067e3dd814f35781ed565a51d1733b70579d2d685b3fbfd2c5904130472c/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546260605
Jul 17 03:25:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:33.948Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-infiniband-umad6.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689546267691
Jul 17 03:25:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:34.008Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"19\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546260605
Jul 17 03:25:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:34.035Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_ready_replicas\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-0-firewall-6789f4bc9b\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546259894
Jul 17 03:25:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:34.072Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-7398125dcac036e9467a5e64699d8816e5ffa907fc39919e5cadea0af235e691-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689546267691
Jul 17 03:25:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:34.089Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:c0-0000:c0:01.1-0000:c1:00.1-net-ens7f1np1.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689546267691
Jul 17 03:25:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:34.095Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-9982afd0\\\\x2d5736\\\\x2d4dce\\\\x2db764\\\\x2d2f7b0c30831a-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2ddx6r4.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689546267691
Jul 17 03:25:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:34.106Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:40-0000:40:01.1-0000:41:00.0-infiniband-mlx5_4.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689546267691
Jul 17 03:25:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:34.118Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"microcode.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"service\"}" timestamp=1689546267691
Jul 17 03:25:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:34.145Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-update-done.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"service\"}" timestamp=1689546267691
Jul 17 03:25:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:34.241Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db15\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546269260
Jul 17 03:25:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:34.258Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"lm_sensors.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"service\"}" timestamp=1689546267691
Jul 17 03:25:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:34.312Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_reads_bytes_total\", asset=\"twa-tsgx05\", asset_id=\"99\", container=\"lb-tcp-443\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546248698
Jul 17 03:25:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:40.537Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas_unavailable\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"prometheus-alertmanager\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546280034
Jul 17 03:25:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:40.554Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"62\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546271647
Jul 17 03:25:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:40.589Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"mlocate-updatedb.timer\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"timer\"}" timestamp=1689546275764
Jul 17 03:25:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:40.608Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-br_dign_c.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689546275764
Jul 17 03:25:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:40.627Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_address_assign_type\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"br_dign_c\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546271647
Jul 17 03:25:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:40.678Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_netstat_IpExt_OutOctets\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546271647
Jul 17 03:25:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:40.692Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_frame_total\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"eno34np1\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546271647
Jul 17 03:25:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:40.817Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_DuplicateKeyCheckTime_max\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"PhoenixIndexer\"}" timestamp=1689546279757
Jul 17 03:25:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:41.053Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"91\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546271647
Jul 17 03:25:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:41.075Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:00-0000:00:01.1-0000:01:00.0-net-ens1f0np0.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689546275764
Jul 17 03:25:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:41.463Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_hbase_table_meta_metric_putBatchTime_25th_percentile\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546279757
Jul 17 03:25:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:42.551Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"tmpfs\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/var/lib/kubelet/pods/e10cda11-8be1-4b90-a6b7-1a1567a187aa/volumes/kubernetes.io~projected/kube-api-access-4zwcp\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546276242
Jul 17 03:25:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:42.567Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"phy_tx_error_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", dev=\"virtio_dign_s\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"mrzcpd-exporter\"}" timestamp=1689546279613
Jul 17 03:25:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:42.600Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_up\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens8f0np0\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546276242
Jul 17 03:25:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:42.611Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_hwmon_temp_celsius\", asset=\"pcap-tsgx01\", asset_id=\"71\", chip=\"0000:80:01_2_0000:82:00_0\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", sensor=\"temp1\", service=\"prometheus-node-exporter\"}" timestamp=1689546276242
Jul 17 03:25:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:42.625Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_mem_not_counted_for_eviction_bytes\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546274838
Jul 17 03:25:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:42.634Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Unknown_Bit\", app_name=\"sapp4\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", vsysId=\"1\"}" timestamp=1689546267581
Jul 17 03:25:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:42.853Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", cpu=\"5\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546276242
Jul 17 03:25:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:42.949Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_io_current\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/sys/fs/cgroup\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546276474
Jul 17 03:25:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:42.995Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"mlocate-updatedb.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"simple\", type=\"service\"}" timestamp=1689546280671
Jul 17 03:25:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:43.016Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", container=\"mrapm-device\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"cadvisor\", failure_type=\"pgfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"container\"}" timestamp=1689546274711
Jul 17 03:25:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:43.609Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_ulimits_soft\", asset=\"pcap-tsgx01\", asset_id=\"71\", container=\"lb-tcp-443\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", ulimit=\"max_open_files\"}" timestamp=1689546279204
Jul 17 03:25:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:43.791Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_terminated\", asset=\"pcap-tsgx01\", asset_id=\"71\", container=\"prometheus-alertmanager-configmap-reload\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-alertmanager-6df7f457b6-m8cl7\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"98daa3ea-45f0-41c2-be94-512fb3e2a016\"}" timestamp=1689546275154
Jul 17 03:25:44 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:44.209Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", cpu=\"72\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546276242
Jul 17 03:25:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:49.265Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_filteredReadRequestCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"session_record_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"88a9bbe2136852f7ad790e6c839f28d2\", table=\"index_time_session_record_hos_bucket\", type=\"index\"}" timestamp=1689546287438
Jul 17 03:25:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:49.558Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_2_metric_majorCompactionTime_98th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546287438
Jul 17 03:25:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:49.643Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_pubsub_channels\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546286625
Jul 17 03:25:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:49.666Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Udp_Link_C2S_ALL\", app_name=\"sapp4\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", vsysId=\"1\"}" timestamp=1689546282447
Jul 17 03:25:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:49.670Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_DuplicateKeyCheckTime_median\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"PhoenixIndexer\"}" timestamp=1689546287438
Jul 17 03:25:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:49.773Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_numFilesCompactedCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"proxy_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"e20b45af9eed7e0a2350cbbb6b88adfd\", table=\"index_filename_proxy_hos_bucket\", type=\"index\"}" timestamp=1689546287438
Jul 17 03:25:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:49.928Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx08\", asset_id=\"105\", container=\"dign-client\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Async\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546287484
Jul 17 03:25:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:49.937Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"prometheus-pushgateway\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"sleeping\"}" timestamp=1689546260806
Jul 17 03:25:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:49.955Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"105\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546286709
Jul 17 03:25:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:49.979Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"telegraf-security\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", state=\"sleeping\"}" timestamp=1689546271164
Jul 17 03:25:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:49.993Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"24\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546286709
Jul 17 03:25:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:50.005Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_labels\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-node-exporter-bz7wf\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"d58b8827-5397-48fd-9ce6-3976a10dcb09\"}" timestamp=1689546279668
Jul 17 03:25:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:50.008Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_frequency_max_hertz\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"67\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546286709
Jul 17 03:25:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:50.048Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"84\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546286709
Jul 17 03:25:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:50.054Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_filteredReadRequestCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"6395a6e0dd130e6dfabcd31c8dcecdf0\", table=\"proxy_hos_bucket\", type=\"data\"}" timestamp=1689546287438
Jul 17 03:25:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:50.064Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db40\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546286625
Jul 17 03:25:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:50.064Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_free_bytes\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"shm\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/1d0a9283baaeb68215db2c28cfe83a4ec2014931320cc1ec38d111b6cd89d7b3/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546286709
Jul 17 03:25:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:50.075Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_changes_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth5b68c36f\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546286709
Jul 17 03:25:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:50.087Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"104\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546286709
Jul 17 03:25:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:50.093Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_hbase_table_rsgroup_metric_deleteBatchTime_25th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546287438
Jul 17 03:25:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:50.182Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-infiniband-umad9.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689546281057
Jul 17 03:25:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:50.234Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"openibd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"service\"}" timestamp=1689546281057
Jul 17 03:25:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:50.247Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-46d147ac1dc6ccb0ad945ee1ab85a16f7457053362dc1b89d248dd448e24319b-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689546281057
Jul 17 03:25:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:50.254Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_writes_total\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"dashboard-metrics-scraper\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546264596
Jul 17 03:25:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:50.359Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_2_metric_flushTime_num_ops\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546287438
Jul 17 03:25:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:50.384Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_commands_processed_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546287556
Jul 17 03:25:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:50.400Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_defrag_key_hits\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546287556
Jul 17 03:25:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:50.420Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"twa-tsgx10\", asset_id=\"109\", cpu=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546281797
Jul 17 03:25:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:50.685Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_scaling_frequency_min_hertz\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"70\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546286709
Jul 17 03:25:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:50.700Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546286709
Jul 17 03:25:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:50.801Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-ba106c44\\\\x2dd5c3\\\\x2d446a\\\\x2db10b\\\\x2d530b2492f73f-volume\\\\x2dsubpaths-config\\\\x2dvolume-firewall-1.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689546281057
Jul 17 03:25:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:50.849Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_file_descriptors\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"dign-server-dns\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546278983
Jul 17 03:25:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:50.880Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-mrzcpd-huge_pages.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689546281057
Jul 17 03:25:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:50.883Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_max_usage_bytes\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"lb-tcp-80\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546260589
Jul 17 03:25:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:50.904Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_read_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/var/lib/kubelet/pods/a0e438ae-62dd-4767-974d-b0821c7c31f3/volumes/kubernetes.io~projected/kube-api-access-tlgrw\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546280806
Jul 17 03:25:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:50.920Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"twa-tsgx08\", asset_id=\"105\", container=\"lb-tcp-80\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"running\"}" timestamp=1689546280628
Jul 17 03:25:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:50.952Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_restarts_total\", asset=\"twa-tsgx08\", asset_id=\"105\", container=\"traefik\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"traefik-64f55bb67d-rh9mv\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"32707ee1-6f45-4d2c-8e0e-78edc233adc3\"}" timestamp=1689546281568
Jul 17 03:25:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:50.968Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_frequency_min_hertz\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"72\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546286709
Jul 17 03:25:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:51.042Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_frequency_max_hertz\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"27\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546286709
Jul 17 03:25:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:51.057Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"118\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546286709
Jul 17 03:25:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:51.095Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_multicast_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens6f1np1\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546286709
Jul 17 03:25:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:51.280Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_io_time_weighted_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"overlay_0-129\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546280806
Jul 17 03:25:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:51.297Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_status_failed\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", job_name=\"helm-install-tsg-traffic-engine-numa-0\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546281568
Jul 17 03:25:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:53.061Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_bytes_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"cadvisor\", interface=\"cnibr0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546287006
Jul 17 03:25:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:53.099Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", cpu=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546289337
Jul 17 03:25:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:53.238Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"62\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546283391
Jul 17 03:25:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:53.257Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx12\", asset_id=\"113\", cpu=\"97\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546283391
Jul 17 03:25:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:53.258Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_flags\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"eno35np2\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546283391
Jul 17 03:25:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:53.277Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"overlay\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-4bd025708f583c4ffee2677091b4b61981ab287dc1df42dabf020acf814abcf6-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689546285496
Jul 17 03:25:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:53.297Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:80-0000:80:01.1-0000:81:00.0-host0-target0:2:0-0:2:0:0-block-sda.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689546285496
Jul 17 03:25:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:53.350Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"ext4\", name=\"data-var-lib-kubelet-pods-a7dd7c83\\\\x2d1f08\\\\x2d4d39\\\\x2da9a0\\\\x2deff0bdbd01be-volume\\\\x2dsubpaths-config\\\\x2dvolume-telegraf\\\\x2dsecurity-2.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689546292135
Jul 17 03:25:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:53.357Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_system_seconds_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", container=\"prometheus-alertmanager-configmap-reload\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546279454
Jul 17 03:25:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:53.433Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"121\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546289337
Jul 17 03:25:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:53.480Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth081275ce\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546283391
Jul 17 03:25:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:53.508Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", container=\"dashboard-metrics-scraper\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Async\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546283281
Jul 17 03:25:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:53.532Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", pod=\"tsg-traffic-engine-numa-0-firewall-7c999f565f-xq5dg\", project=\"TSG-OS\", project_id=\"7\", reason=\"NodeLost\", service=\"prometheus-kube-state-metrics\", uid=\"93005333-2bbb-4ce3-a6b6-a5b632269832\"}" timestamp=1689546284478
Jul 17 03:25:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:53.561Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_max_usage_bytes\", asset=\"msh-tsgx12\", asset_id=\"113\", container=\"dign-client\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546265770
Jul 17 03:25:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:53.591Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"87\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546289337
Jul 17 03:25:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:53.628Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_ulimits_soft\", asset=\"msh-tsgx12\", asset_id=\"113\", container=\"dign-server-dns\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", ulimit=\"max_open_files\"}" timestamp=1689546276989
Jul 17 03:25:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:53.631Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db62\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546278659
Jul 17 03:25:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:53.646Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db106\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546278659
Jul 17 03:25:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:53.648Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_metadata_resource_version\", asset=\"msh-tsgx12\", asset_id=\"113\", configmap=\"kube-root-ca.crt\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"consul\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546288663
Jul 17 03:25:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:53.655Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"69\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546283391
Jul 17 03:25:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:53.671Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx12\", asset_id=\"113\", cpu=\"28\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546283391
Jul 17 03:25:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:53.678Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_info\", asset=\"pcap-tsgx07\", asset_id=\"85\", configmap=\"sce-tsg-traffic-engine-numa-0\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546282058
Jul 17 03:25:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:53.688Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_process_id\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546278659
Jul 17 03:25:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:53.694Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Unknown\", pod=\"helm-install-tsg-traffic-engine-numa-0-clqgr\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"1ea39cdb-8ff7-4292-b608-b56268d85d12\"}" timestamp=1689546282058
Jul 17 03:25:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:53.741Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx12\", asset_id=\"113\", cpu=\"13\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546283391
Jul 17 03:25:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:53.804Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_vl15_dropped_total\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_4\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546283391
Jul 17 03:25:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:53.826Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth0b89e318\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546289337
Jul 17 03:25:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:53.830Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"proc-sys-fs-binfmt_misc.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689546291179
Jul 17 03:25:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:53.833Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db171\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546278659
Jul 17 03:25:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:53.851Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_max_fds\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"irqbalance.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689546291179
Jul 17 03:25:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:53.854Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db198\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546279367
Jul 17 03:25:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:53.860Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"msh-tsgx12\", asset_id=\"113\", cpu=\"23\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546283391
Jul 17 03:25:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:53.877Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"iscsi-shutdown.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"oneshot\", type=\"service\"}" timestamp=1689546285496
Jul 17 03:25:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:53.882Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2did-scsi\\\\x2d362cea7f0add59d002c2c317c692f9f02\\\\x2dpart4.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689546291179
Jul 17 03:25:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:53.927Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"var-lib-kubelet-pods-a70ba7d6\\\\x2dc83e\\\\x2d453e\\\\x2d962c\\\\x2df10b0073510f-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2dklfkn.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689546291179
Jul 17 03:25:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:53.979Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"msh-tsgx12\", asset_id=\"113\", container=\"lb-tcp-80\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"cadvisor\", failure_type=\"pgfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"hierarchy\"}" timestamp=1689546265425
Jul 17 03:25:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:53.997Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_packets_dropped_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"cadvisor\", interface=\"ens3f1np1\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546290420
Jul 17 03:25:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:54.026Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db133\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546278659
Jul 17 03:25:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:54.033Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dracut-cmdline.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"oneshot\", type=\"service\"}" timestamp=1689546285496
Jul 17 03:25:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:54.041Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-8ba148c1\\\\x2df25f\\\\x2d4fa4\\\\x2d82b3\\\\x2d03b9d5a8ace6-volume\\\\x2dsubpaths-config\\\\x2dvolume-telegraf-2.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689546291179
Jul 17 03:25:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:54.211Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Unknown\", pod=\"traefik-64f55bb67d-bcw8n\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"b9686156-a602-47ce-b798-d231f10c8f92\"}" timestamp=1689546282058
Jul 17 03:25:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:54.229Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_annotations\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-0-firewall-5b4b8cbcc6\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546282058
Jul 17 03:25:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:54.250Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_daemonset_status_number_ready\", asset=\"msh-tsgx12\", asset_id=\"113\", daemonset=\"dign-client\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546288663
Jul 17 03:25:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:54.271Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_address_available\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_endpoint=\"shaping-announce-port-tsg-traffic-engine-numa-0\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546288663
Jul 17 03:25:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:54.292Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cpu=\"113\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546279409
Jul 17 03:25:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:54.314Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cpu=\"15\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546279409
Jul 17 03:25:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:54.323Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_spec_replicas\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"prometheus-alertmanager-6df7f457b6\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546288663
Jul 17 03:25:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:54.422Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dracut-shutdown-onfailure.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"oneshot\", type=\"service\"}" timestamp=1689546280884
Jul 17 03:25:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:54.505Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cpu=\"36\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546279409
Jul 17 03:25:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:54.576Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"msh-tsgx08\", asset_id=\"65\", container=\"prometheus-server\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"cadvisor\", failure_type=\"pgfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"hierarchy\"}" timestamp=1689546269521
Jul 17 03:25:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:54.657Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-veth8ec198c4.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689546291179
Jul 17 03:25:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:54.912Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_mapped_file\", asset=\"pcap-tsgx07\", asset_id=\"85\", container=\"exporter\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546279669
Jul 17 03:25:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:54.939Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_terminated\", asset=\"pcap-tsgx07\", asset_id=\"85\", container=\"dign-server-dns\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"dign-server-4j9wh\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"70d61467-349b-47d9-b813-5d6a3af28cef\"}" timestamp=1689546282058
Jul 17 03:25:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:55.303Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_terminated\", asset=\"pcap-tsgx07\", asset_id=\"85\", container=\"lb-tcp-443\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"svclb-traefik-b613548a-2ndxt\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"312ba98a-2da4-49c6-9feb-0611d5586d06\"}" timestamp=1689546282058
Jul 17 03:25:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:55.327Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_info\", asset=\"pcap-tsgx07\", asset_id=\"85\", cluster_ip=\"10.43.0.1\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"kubernetes\", uid=\"cea73bef-c295-485d-97b8-703228b4b49a\"}" timestamp=1689546282058
Jul 17 03:25:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:55.358Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_info\", address=\"b8:ce:f6:88:53:b7\", asset=\"pcap-tsgx07\", asset_id=\"85\", broadcast=\"ff:ff:ff:ff:ff:ff\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens2f1np1\", duplex=\"unknown\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operstate=\"down\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546279409
Jul 17 03:25:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:55.384Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_bytes_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens8f0np0\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546279409
Jul 17 03:25:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:56.862Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", container=\"telegraf\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Read\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689546284736
Jul 17 03:25:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:57.004Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"58\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546285883
Jul 17 03:25:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:57.030Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_persistentvolume_created\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", persistentvolume=\"pvc-ef949bcc-8cbc-4c60-88fa-1e99971cf7bb\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546293389
Jul 17 03:25:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:57.168Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_readonly\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"shm\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/027350f54bb89061a60d10b405de81d85cf1dac7e54e96d6ac48c11ceaf660cd/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546285883
Jul 17 03:25:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:57.185Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_hwmon_sensor_label\", asset=\"pcap-tsgx03\", asset_id=\"73\", chip=\"pci0000:00_0000:00:19_3\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", label=\"tccd4\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", sensor=\"temp6\", service=\"prometheus-node-exporter\"}" timestamp=1689546285883
Jul 17 03:25:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:57.297Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_info\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", version=\"go1.17.3\"}" timestamp=1689546285883
Jul 17 03:25:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:57.318Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_labels\", asset=\"pcap-tsgx03\", asset_id=\"73\", configmap=\"shaping-tsg-traffic-engine-numa-0\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546293389
Jul 17 03:25:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:57.463Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_info\", asset=\"pcap-tsgx03\", asset_id=\"73\", configmap=\"sce-tsg-traffic-engine-numa-0\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546293389
Jul 17 03:25:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:57.483Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-46cc56338cc2b9e89fbf7d973167e34db2b38925f52e228e15911ebb28736604-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689546283372
Jul 17 03:25:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:57.578Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_created\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", job_name=\"helm-install-traefik-crd\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546293389
Jul 17 03:25:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:57.952Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_disk_written_bytes_total\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"sdc\", endpoint=\"OLAP-Node-Exporter-pcap-dellr740-dt001\", endpoint_id=\"251\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546297557
Jul 17 03:25:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:57.980Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_scaling_frequency_hertz\", asset=\"msh-dellr740-dt003\", asset_id=\"9\", cpu=\"11\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-dt003\", endpoint_id=\"229\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"hbase_master,clickhouse_data,kafka,gohangout,zookeeper,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546297407
Jul 17 03:25:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:59.420Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_scaling_frequency_min_hertz\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", cpu=\"30\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"OLAP-Node-Exporter-pcap-dellr740-dt001\", endpoint_id=\"251\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546297557
Jul 17 03:25:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:59.717Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_avail_bytes\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"shm\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/466704d5b4434a33850111b8e122e1007e0a7a1d3f6e50fd81b1ce0bede9b8a2/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546289083
Jul 17 03:25:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:59.838Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_cpu_shares\", asset=\"msh-tsgx01\", asset_id=\"51\", container=\"metrics\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546294593
Jul 17 03:25:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:59.853Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_ready_replicas\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-1-firewall-6c6f4bc877\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546287425
Jul 17 03:25:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:59.872Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"123\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546289083
Jul 17 03:25:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:25:59.954Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_info\", address=\"56:6a:e1:ce:6f:2c\", asset=\"msh-tsgx01\", asset_id=\"51\", broadcast=\"ff:ff:ff:ff:ff:ff\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth9b710207\", duplex=\"full\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operstate=\"up\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546289083
Jul 17 03:26:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:00.072Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_namespace_annotations\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-public\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546287425
Jul 17 03:26:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:00.159Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-9b4a3cb86d4218901fde5bed69b49773de02a8dbc0d6c10ded05a09b412d035d.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"scope\"}" timestamp=1689546284879
Jul 17 03:26:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:00.213Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"77\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546289083
Jul 17 03:26:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:00.232Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"61\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546289083
Jul 17 03:26:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:00.267Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"75\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546289083
Jul 17 03:26:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:00.271Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_commands_duration_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cmd=\"config\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546285071
Jul 17 03:26:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:00.596Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_namespace_status_phase\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", phase=\"Active\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546287425
Jul 17 03:26:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:00.610Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"msh-tsgx01\", asset_id=\"51\", condition=\"unknown\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-server-6ff784d758-x2kns\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"7e3e09eb-7bd8-47c8-bd0d-9b8a588f34f3\"}" timestamp=1689546287425
Jul 17 03:26:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:00.696Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"94\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546289083
Jul 17 03:26:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:00.729Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db253\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546285071
Jul 17 03:26:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:01.306Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_labels\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-0-firewall-57597b866c\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546287425
Jul 17 03:26:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:02.479Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_info\", address=\"66:eb:b2:b9:c4:73\", asset=\"msh-tsgx02\", asset_id=\"63\", broadcast=\"ff:ff:ff:ff:ff:ff\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"virtio_dign_c\", duplex=\"full\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operstate=\"unknown\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546298318
Jul 17 03:26:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:02.615Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_usage_seconds_total\", asset=\"msh-tsgx02\", asset_id=\"63\", container=\"prometheus-server\", cpu=\"total\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546275982
Jul 17 03:26:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:02.642Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_owner\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", job_name=\"helm-install-configmap-reloader\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", owner_is_controller=\"<none>\", owner_kind=\"<none>\", owner_name=\"<none>\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546291412
Jul 17 03:26:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:02.659Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"cm-policy-local-cache-redis-master-69967b5879-h2tzb\", project=\"TSG-OS\", project_id=\"7\", reason=\"NodeLost\", service=\"prometheus-kube-state-metrics\", uid=\"695dd1c7-6904-4ca3-aa3b-49efdba3b420\"}" timestamp=1689546291412
Jul 17 03:26:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:02.989Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_scaling_frequency_hertz\", asset=\"pcap-dellr740-dt002\", asset_id=\"33\", cpu=\"26\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"OLAP-Node-Exporter-pcap-dellr740-dt002\", endpoint_id=\"253\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546302497
Jul 17 03:26:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:03.023Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_scaling_frequency_min_hertz\", asset=\"pcap-dellr740-dt002\", asset_id=\"33\", cpu=\"16\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"OLAP-Node-Exporter-pcap-dellr740-dt002\", endpoint_id=\"253\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546302497
Jul 17 03:26:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:03.027Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.grpc.v1.cri-sandboxes-fbc8bfa35da10a5bb4fd9ef38b09e477d7688185bad9d1e6aa07344a55103575-shm.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689546288447
Jul 17 03:26:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:03.043Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"prometheus_target_metadata_cache_bytes\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", instance=\"10.10.20.159:10090\", scrape_job=\"asset_ping_89\", type=\"global\"}" timestamp=1689546302994
Jul 17 03:26:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:03.065Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-48fe8066\\\\x2db792\\\\x2d469f\\\\x2da7b4\\\\x2df5bfac9a3865-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2d9cvbl.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689546288447
Jul 17 03:26:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:03.085Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_mcache_inuse_bytes\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-dt001\", endpoint_id=\"225\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546302988
Jul 17 03:26:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:03.133Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", cpu=\"15\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-dt001\", endpoint_id=\"225\", mode=\"user\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546302988
Jul 17 03:26:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:03.142Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-logind.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689546288447
Jul 17 03:26:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:03.360Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_scrape_collector_duration_seconds\", asset=\"msh-tsgx02\", asset_id=\"63\", collector=\"sockstat\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546298318
Jul 17 03:26:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:03.370Z caller=dedupe.go:112 component=remote level=info remote_name=52db6f url=http://10.10.10.159:10090/nz-agent/prometheus/proxy/api/v1/write msg="Dropped sample for series that was not explicitly dropped via relabelling" ref=4436650
Jul 17 03:26:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:03.441Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-e2ed79069ab214e078f0c5720399a08b070883716cce978eb2892becb7721b47.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"scope\"}" timestamp=1689546288447
Jul 17 03:26:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:03.454Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2dpartlabel-TSG\\\\x2dOS\\\\x2dDATA.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689546288447
Jul 17 03:26:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:03.476Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx02\", asset_id=\"63\", cpu=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546298318
Jul 17 03:26:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:03.485Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"mlocate-updatedb.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"service\"}" timestamp=1689546288447
Jul 17 03:26:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:03.502Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Running\", pod=\"helm-install-traefik-crd-99hbv\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"10c928ff-b1be-480e-97e1-363fb468914b\"}" timestamp=1689546291412
Jul 17 03:26:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:03.653Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_frame_total\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"em2\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-dt001\", endpoint_id=\"225\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546302988
Jul 17 03:26:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:03.667Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"net_conntrack_dialer_conn_failed_total\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", dialer_name=\"35\", instance=\"10.10.20.159:10090\", reason=\"resolution\", type=\"global\"}" timestamp=1689546302994
Jul 17 03:26:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:03.838Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link_mode\", asset=\"msh-dellr740-olap005\", asset_id=\"19\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"em4\", endpoint=\"NC-Spark-msh-dellr740-olap005\", endpoint_id=\"17\", module=\"NC-Spark\", module_id=\"47\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546303391
Jul 17 03:26:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:03.974Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"net_conntrack_dialer_conn_failed_total\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", dialer_name=\"343\", instance=\"10.10.20.159:10090\", reason=\"resolution\", type=\"global\"}" timestamp=1689546302994
Jul 17 03:26:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:04.054Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"prometheus_target_metadata_cache_entries\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", instance=\"10.10.20.159:10090\", scrape_job=\"17\", type=\"global\"}" timestamp=1689546302994
Jul 17 03:26:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:06.571Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"druid_supervisor_healthy_num\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Spark-msh-dellr740-olap003\", endpoint_id=\"13\", module=\"NC-Spark\", module_id=\"47\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546306093
Jul 17 03:26:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:06.731Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", cpu=\"31\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Spark-msh-dellr740-olap003\", endpoint_id=\"13\", mode=\"softirq\", module=\"NC-Spark\", module_id=\"47\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546306093
Jul 17 03:26:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:06.744Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_annotations\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_endpoint=\"kube-dns\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546301227
Jul 17 03:26:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:06.987Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_hwmon_chip_names\", asset=\"twa-tsgx01\", asset_id=\"91\", chip=\"0000:60:03_1_0000:63:00_1\", chip_name=\"bnxt_en\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546303910
Jul 17 03:26:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:06.997Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_usage_bytes\", asset=\"twa-tsgx01\", asset_id=\"91\", container=\"exporter-merger\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546297900
Jul 17 03:26:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:07.003Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_packets_received_total\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_2\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546303910
Jul 17 03:26:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:07.017Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"twa-tsgx01\", asset_id=\"91\", condition=\"true\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", pod=\"dashboard-metrics-scraper-5b94897dc9-xrbx4\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"d73900af-89f6-4478-880d-06274d9e40b5\"}" timestamp=1689546301227
Jul 17 03:26:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:07.030Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_drop_total\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth2d0ed10f\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546303910
Jul 17 03:26:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:07.033Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_memory_HugePages_Surp\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546303910
Jul 17 03:26:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:07.134Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"getty-pre.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"target\"}" timestamp=1689546293582
Jul 17 03:26:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:07.222Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_multicast_total\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens6f1np1\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546303910
Jul 17 03:26:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:07.273Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_load1\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546303910
Jul 17 03:26:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:07.310Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cpu=\"19\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546303910
Jul 17 03:26:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:07.349Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-45cfa87f\\\\x2d48cf\\\\x2d44fc\\\\x2da587\\\\x2d24a36ba4ecdb-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2dg2w2t.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689546293582
Jul 17 03:26:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:07.371Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_max_usage_bytes\", asset=\"twa-tsgx01\", asset_id=\"91\", container=\"dign-server-web\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546295740
Jul 17 03:26:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:07.401Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_complete\", asset=\"twa-tsgx01\", asset_id=\"91\", condition=\"false\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", job_name=\"helm-install-configmap-reloader\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546301227
Jul 17 03:26:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:07.536Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_frequency_max_hertz\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", cpu=\"23\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Spark-msh-dellr740-olap003\", endpoint_id=\"13\", module=\"NC-Spark\", module_id=\"47\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546306093
Jul 17 03:26:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:07.549Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_replicas\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-0-firewall-79fdc6d6cf\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546301227
Jul 17 03:26:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:07.574Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_id\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens8f0np0\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546303910
Jul 17 03:26:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:07.590Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:20-0000:20:03.1-0000:21:00.1-infiniband_mad-umad3.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689546293582
Jul 17 03:26:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:07.912Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cpu=\"60\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546303910
Jul 17 03:26:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:07.942Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cpu=\"75\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546303910
Jul 17 03:26:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:08.003Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-modules-load.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"service\"}" timestamp=1689546293582
Jul 17 03:26:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:08.017Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_spec_replicas\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"prometheus-pushgateway\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546301227
Jul 17 03:26:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:08.282Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_down_changes_total\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens8f0np0\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546303910
Jul 17 03:26:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:08.336Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"nsfs\", name=\"run-netns-cni\\\\x2d3562d380\\\\x2da990\\\\x2d80bb\\\\x2d9a8b\\\\x2df7050530dbb4.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689546293582
Jul 17 03:26:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:08.439Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cpu=\"73\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546303910
Jul 17 03:26:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:08.668Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-veth2d0ed10f.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689546293582
Jul 17 03:26:09 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:09.142Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"scrape_series_added\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-Zookeeper-msh-dellr740-dt002\", endpoint_id=\"113\", instance=\"192.168.20.194:9902\", job=\"113\", module=\"DC-Zookeeper\", module_id=\"7\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546308156
Jul 17 03:26:09 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:09.805Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_annotations\", asset=\"msh-tsgx03\", asset_id=\"59\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx03\", endpoint_id=\"427\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"1\", pod=\"helm-install-traefik-crd-mfvl8\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"7e8ea2ff-bd2f-46ed-bd51-38fb3bb9888e\"}" timestamp=1689546295765
Jul 17 03:26:09 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:09.866Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_maxStoreFileAge\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"session_record_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"8624285d7679e25fa47769ce6fc2c350\", table=\"index_filename_session_record_hos_bucket\", type=\"index\"}" timestamp=1689546308804
Jul 17 03:26:09 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:09.884Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_maxCompactionQueueSize\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"firewall_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"667c1822ce0b15155897b30d3dd3335e\", table=\"index_partfile_firewall_hos_bucket\", type=\"index\"}" timestamp=1689546308804
Jul 17 03:26:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:10.405Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_numReferenceFiles\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"dfa2d065ee026585e9af86e1e567c425\", table=\"firewall_hos_bucket\", type=\"data\"}" timestamp=1689546308804
Jul 17 03:26:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:10.509Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_avgStoreFileAge\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"3dfef7d00f6dec45b0db11d38cfa59c5\", table=\"default\", type=\"data\"}" timestamp=1689546308804
Jul 17 03:26:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:10.761Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_numBytesCompactedCount\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"proxy_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"3f3a8f4c87c05c3707c187b169923a9f\", table=\"index_filename_proxy_hos_bucket\", type=\"index\"}" timestamp=1689546308804
Jul 17 03:26:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:11.324Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_1_metric_splitTime_99_9th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546310147
Jul 17 03:26:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:11.328Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_194_4_metric_minStoreFileAge\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546310147
Jul 17 03:26:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:11.360Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_2_metric_compactionInputSize_98th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546310147
Jul 17 03:26:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:11.364Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_putCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"default\", project=\"OLAP\", project_id=\"3\", region=\"0c7e6f72ef2e42287320a6bde3b38821\", table=\"index_filename_default\", type=\"index\"}" timestamp=1689546310147
Jul 17 03:26:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:11.374Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_STATS_metric_putTime_median\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546310147
Jul 17 03:26:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:11.377Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_3_metric_majorCompactionOutputFileCount_90th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546310147
Jul 17 03:26:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:11.397Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_193_1_metric_tableWriteQueryPerSecond_mean_rate\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546310147
Jul 17 03:26:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:11.417Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_LOG_metric_scanTime_mean\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546310147
Jul 17 03:26:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:11.426Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_compactionsQueuedCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"proxy_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"7bcf63599704330b114c0c7e0332621c\", table=\"index_time_proxy_hos_bucket\", type=\"index\"}" timestamp=1689546310147
Jul 17 03:26:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:11.466Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_4_metric_scanSize_min\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546310147
Jul 17 03:26:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:11.489Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_194_3_metric_getTime_max\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546310147
Jul 17 03:26:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:11.493Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_filteredReadRequestCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"84eff2a766cab8f7c830d202f3a96c6e\", table=\"firewall_hos_bucket\", type=\"data\"}" timestamp=1689546310147
Jul 17 03:26:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:11.515Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_maxFlushQueueSize\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"proxy_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"a6078361d4a0ffaafa787562c158d6fc\", table=\"index_time_proxy_hos_bucket\", type=\"index\"}" timestamp=1689546310147
Jul 17 03:26:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:11.599Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_maxFlushQueueSize\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"session_record_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"27e64d59c20926b648fd0e4254c5cf70\", table=\"index_partfile_session_record_hos_bucket\", type=\"index\"}" timestamp=1689546310147
Jul 17 03:26:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:11.618Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_numFilesCompactedCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"knowledge_base_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"04ab4d5780512f01a21447ac053d6154\", table=\"index_filename_knowledge_base_hos_bucket\", type=\"index\"}" timestamp=1689546310147
Jul 17 03:26:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:11.692Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_tsg_table_report_result_metric_compactionOutputFileCount_min\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546310147
Jul 17 03:26:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:11.711Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_getCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"rtp_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"79bb27e5401722645342c5e31b05b0ec\", table=\"index_time_rtp_hos_bucket\", type=\"index\"}" timestamp=1689546310147
Jul 17 03:26:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:11.768Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_2_metric_compactionInputSize_25th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546310147
Jul 17 03:26:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:11.782Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_1_metric_tableSize\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546310147
Jul 17 03:26:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:11.795Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_compactionsQueuedCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"troubleshooting_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"dd829c862a6157803d65b4e2d0e75ad1\", table=\"index_time_troubleshooting_hos_bucket\", type=\"index\"}" timestamp=1689546310147
Jul 17 03:26:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:11.814Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_MajorCompactionInputFileCount_99th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Server\"}" timestamp=1689546310147
Jul 17 03:26:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:11.841Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_2_metric_tableWriteQueryPerSecond_5min_rate\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546310147
Jul 17 03:26:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:11.909Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_numFilesCompactedCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"knowledge_base_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"005a8abb9a4cffc600e44c8a98576e58\", table=\"index_partfile_knowledge_base_hos_bucket\", type=\"index\"}" timestamp=1689546310147
Jul 17 03:26:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:11.986Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_compactionsQueuedCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"assessment_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"db3245bccdc6403a0161f317bbd97ed6\", table=\"index_filename_assessment_hos_bucket\", type=\"index\"}" timestamp=1689546310147
Jul 17 03:26:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:11.998Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_session_record_hos_bucket_metric_compactionOutputFileCount_99_9th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546310147
Jul 17 03:26:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:12.005Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_3_metric_majorCompactionInputSize_25th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546310147
Jul 17 03:26:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:12.045Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_CHILD_LINK_metric_putBatchTime_95th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546310147
Jul 17 03:26:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:12.060Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_avgStoreFileAge\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"firewall_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"9c4bfda458c8dfa19186629bdfb95731\", table=\"index_partfile_firewall_hos_bucket\", type=\"index\"}" timestamp=1689546310147
Jul 17 03:26:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:12.199Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_193_1_metric_majorCompactionInputFileCount_99_9th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546310147
Jul 17 03:26:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:12.212Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_4_metric_majorCompactionInputFileCount_median\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546310147
Jul 17 03:26:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:12.349Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_session_record_hos_bucket_metric_compactionInputSize_75th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546310147
Jul 17 03:26:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:12.400Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_flushesQueuedCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"rtp_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"e195934ff82ce524a5c833d64622bb84\", table=\"index_filename_rtp_hos_bucket\", type=\"index\"}" timestamp=1689546310147
Jul 17 03:26:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:12.457Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_4_metric_majorCompactionOutputSize_95th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546310147
Jul 17 03:26:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:12.475Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_1_metric_compactionInputSize_75th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546310147
Jul 17 03:26:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:12.489Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_STATS_metric_appendTime_95th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546310147
Jul 17 03:26:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:12.569Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_AppendTime_98th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"WAL\"}" timestamp=1689546310147
Jul 17 03:26:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:12.773Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_193_1_metric_compactionOutputFileCount_mean\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546310147
Jul 17 03:26:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:12.792Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_tsg_table_report_result_metric_incrementTime_99th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546310147
Jul 17 03:26:13 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:13.083Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_TASK_metric_scanTime_99_9th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546310147
Jul 17 03:26:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:15.267Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"twa-tsgx07\", asset_id=\"103\", container=\"kube-multus\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"cadvisor\", failure_type=\"pgfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"container\"}" timestamp=1689546307016
Jul 17 03:26:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:15.297Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"twa-tsgx07\", asset_id=\"103\", container=\"exporter\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"sleeping\"}" timestamp=1689546301657
Jul 17 03:26:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:15.321Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_state_started\", asset=\"twa-tsgx07\", asset_id=\"103\", container=\"ipmi-exporter\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"ipmi-exporter-6f8464dd94-54bzn\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"030260b7-67cb-4efd-9b0b-ed9f7bff5dfa\"}" timestamp=1689546308076
Jul 17 03:26:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:15.343Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"102\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546301230
Jul 17 03:26:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:15.373Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"70\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546301230
Jul 17 03:26:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:15.393Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Udp_Offload_Bit\", app_name=\"sapp4\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", vsysId=\"1\"}" timestamp=1689546311690
Jul 17 03:26:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:15.396Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"87\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546301230
Jul 17 03:26:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:15.396Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_up\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"br_dign_s\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546301230
Jul 17 03:26:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:15.419Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"tmpfs\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/var/lib/kubelet/pods/d75737d8-d596-4b86-8834-e9b0de808309/volumes/kubernetes.io~projected/kube-api-access-9549z\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546301230
Jul 17 03:26:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:15.476Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"73\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546301230
Jul 17 03:26:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:15.546Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_created\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"kubernetes-dashboard-csrf\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546308076
Jul 17 03:26:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:15.743Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_size_bytes\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"shm\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/5cc535b1876c0bcb3d5bfc70b104a2eda0ff079147786ef074810b0fedb27237/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546301230
Jul 17 03:26:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:15.837Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas_unavailable\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"cm-policy-local-cache-redis-master\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546308076
Jul 17 03:26:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:15.877Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"twa-tsgx07\", asset_id=\"103\", condition=\"false\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"cm-policy-local-cache-redis-master-69967b5879-p4gsc\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"d6907218-5827-4019-84f7-b49fadb75836\"}" timestamp=1689546308076
Jul 17 03:26:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:15.896Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"twa-tsgx07\", asset_id=\"103\", condition=\"unknown\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-mrzcpd-exporter-6fw7r\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"1714718b-090b-4e95-bbad-3fb0be9d0099\"}" timestamp=1689546308076
Jul 17 03:26:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:15.988Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"127\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546301230
Jul 17 03:26:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:16.031Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"25\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546301230
Jul 17 03:26:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:16.050Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_packets_received_total\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_1\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546301230
Jul 17 03:26:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:16.064Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546301230
Jul 17 03:26:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:16.078Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"68\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546301230
Jul 17 03:26:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:16.172Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_complete\", asset=\"twa-tsgx07\", asset_id=\"103\", condition=\"unknown\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", job_name=\"helm-install-traefik\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546308076
Jul 17 03:26:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:16.211Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"100\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546301230
Jul 17 03:26:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:16.231Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db94\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546306027
Jul 17 03:26:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:16.247Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"scrape_series_added\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_instance=\"10.42.0.65:8080\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546308076
Jul 17 03:26:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:16.293Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"67\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546301230
Jul 17 03:26:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:16.376Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_annotations\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"metrics-server-54dc485875-cw4sf\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"a203ebf3-8366-4cfd-b036-168e793673ce\"}" timestamp=1689546308076
Jul 17 03:26:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:16.388Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Pending\", pod=\"svclb-traefik-b947eccc-g6kks\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"1b7e4345-c37e-48b7-8d02-089957a16f23\"}" timestamp=1689546308076
Jul 17 03:26:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:16.456Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_oom_events_total\", asset=\"twa-tsgx07\", asset_id=\"103\", container=\"metrics-server\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546299962
Jul 17 03:26:18 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:18.794Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"java_lang_GarbageCollector_LastGcInfo_memoryUsageBeforeGc_init\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-Yarn-pcap-dellr740-dt001-NodeManager\", endpoint_id=\"357\", key=\"Compressed Class Space\", module=\"DC-Yarn\", module_id=\"1\", name=\"PS Scavenge\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546317154
Jul 17 03:26:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:19.420Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_state_started\", asset=\"twa-tsgx06\", asset_id=\"101\", container=\"telegraf-security\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", pod=\"tsg-traffic-engine-numa-0-firewall-5db5dbc44c-866zl\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"9145d365-24c7-4ff1-a1ee-0a800b33502b\"}" timestamp=1689546313949
Jul 17 03:26:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:19.421Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"7\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546317612
Jul 17 03:26:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:19.441Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_info\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.traefik.v1\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546313949
Jul 17 03:26:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:19.452Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-f7154d71794935019fb20b48fedfc413684b1669bd9e3a0cb6f8904135d1d212-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689546304356
Jul 17 03:26:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:19.465Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_up_changes_total\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth310fe810\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546317612
Jul 17 03:26:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:19.469Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"var-lib-kubelet-pods-9145d365\\\\x2d24c7\\\\x2d4ff1\\\\x2da1ee\\\\x2d0a800b33502b-volume\\\\x2dsubpaths-config\\\\x2dvolume-telegraf-2.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689546304356
Jul 17 03:26:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:19.592Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_working_set_bytes\", asset=\"twa-tsgx06\", asset_id=\"101\", container=\"dign-server-dns\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546306905
Jul 17 03:26:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:19.608Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"twa-tsgx06\", asset_id=\"101\", condition=\"Available\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"traefik\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"false\"}" timestamp=1689546313949
Jul 17 03:26:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:19.649Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Pending\", pod=\"helm-install-configmap-reloader-pmpzk\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"86d10a1f-8af3-4a7c-b9f6-a63e3e9ca659\"}" timestamp=1689546313949
Jul 17 03:26:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:19.901Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2did-scsi\\\\x2dSDELL_PERC_H345_Adp_0046807ed21cf8262c00758284f0a7ce\\\\x2dpart1.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689546304356
Jul 17 03:26:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:20.007Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"60\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546317612
Jul 17 03:26:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:20.319Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_usage_bytes\", asset=\"twa-tsgx06\", asset_id=\"101\", container=\"firewall\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689546312872
Jul 17 03:26:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:20.325Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx06\", asset_id=\"101\", container=\"firewall\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Total\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689546312872
Jul 17 03:26:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:20.348Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"twa-tsgx06\", asset_id=\"101\", condition=\"false\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"svclb-traefik-2b43699a-xnrnl\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"d430cb8f-d3b8-4233-a528-300590ccd3d3\"}" timestamp=1689546313949
Jul 17 03:26:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:20.382Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"machine_memory_bytes\", asset=\"twa-tsgx06\", asset_id=\"101\", boot_id=\"4cc29224-84b8-4ede-9e54-ad09bb97a1c8\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"cadvisor\", machine_id=\"f4f4243e37354cd8ad90c0e5fb4d876c\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", system_uuid=\"4c4c4544-005a-3210-8032-b3c04f5a4333\"}" timestamp=1689546312894
Jul 17 03:26:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:20.398Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"twa-tsgx06\", asset_id=\"101\", condition=\"Progressing\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"tsg-traffic-engine-numa-1-firewall\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"unknown\"}" timestamp=1689546313949
Jul 17 03:26:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:20.453Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_labels\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"ipmi-exporter\", uid=\"c8512b2e-e307-4a03-bd90-9ddf5fbeff82\"}" timestamp=1689546313949
Jul 17 03:26:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:20.774Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"61\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546317612
Jul 17 03:26:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:21.007Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_terminated\", asset=\"pcap-tsgx10\", asset_id=\"89\", container=\"redis\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"cm-policy-local-cache-redis-master-69967b5879-8s2nz\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"efdb7e93-9e91-4d51-b008-336f3a7336fb\"}" timestamp=1689546313828
Jul 17 03:26:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:21.025Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"pcap-tsgx10\", asset_id=\"89\", condition=\"false\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"dign-client-g7llw\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"b7bee1cc-245a-40f5-a87e-4563cb9edfda\"}" timestamp=1689546313828
Jul 17 03:26:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:21.086Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"42\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546317612
Jul 17 03:26:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:21.154Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"43\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546308263
Jul 17 03:26:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:21.277Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546308263
Jul 17 03:26:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:21.281Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_threads_max\", asset=\"twa-tsgx06\", asset_id=\"101\", container=\"prometheus-alertmanager\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546306923
Jul 17 03:26:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:21.302Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-a3608d959944782f099e0cf6bc1391a8c417fb48f20f8f25793ab006081e3c14-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689546304356
Jul 17 03:26:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:21.363Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_packets_total\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"cadvisor\", interface=\"ens4f2np2\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546310169
Jul 17 03:26:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:21.366Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_address_assign_type\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens8f1np1\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546317612
Jul 17 03:26:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:21.375Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_errors_received_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"mlx5_6\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546308263
Jul 17 03:26:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:21.480Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_receive_switch_relay_errors_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"mlx5_8\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546308263
Jul 17 03:26:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:21.501Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"10\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546308263
Jul 17 03:26:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:21.614Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_mod_info\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"github.com/golang/protobuf\", nz_agent_id=\"3\", program=\"systemd_exporter\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", version=\"v1.3.5\"}" timestamp=1689546310657
Jul 17 03:26:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:21.816Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"6\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546308263
Jul 17 03:26:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:21.832Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_annotations\", asset=\"pcap-tsgx10\", asset_id=\"89\", configmap=\"kube-root-ca.crt\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546313828
Jul 17 03:26:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:21.844Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_fully_labeled_replicas\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-1-firewall-5d7bfbbbfb\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546313828
Jul 17 03:26:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:21.935Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_writes_merged_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"overlay_0-47\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546312565
Jul 17 03:26:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:21.963Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_reservation_limit_bytes\", asset=\"pcap-tsgx10\", asset_id=\"89\", container=\"configmap-reloader\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546315735
Jul 17 03:26:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:22.034Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-netns-cni\\\\x2d9cef2b7e\\\\x2d4364\\\\x2d37a1\\\\x2d343b\\\\x2d1732bb7d1b06.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689546310657
Jul 17 03:26:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:22.076Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_size_bytes\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"shm\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/219012e52aedd3297447f89162fe8b51fe5cc898f7483af85df1b649a58054f9/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546308263
Jul 17 03:26:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:22.089Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_replication_backlog_bytes\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546314840
Jul 17 03:26:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:23.751Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_size_bytes\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"shm\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/50568b47de6d5e587f8512a9f733c7bb7d4f4e270a38f018a817f2b012648f1d/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546314790
Jul 17 03:26:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:24.037Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_device_id\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth0c6b6d72\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546314790
Jul 17 03:26:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:24.251Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx09\", asset_id=\"55\", cpu=\"45\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546314790
Jul 17 03:26:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:24.757Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-7d4ec355dcf548cd6458ee725c70b6853dcda677cfe9d499d5b8019f4ee5a86c-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689546311102
Jul 17 03:26:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:24.771Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas_updated\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"coredns\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546311122
Jul 17 03:26:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:24.959Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_cache\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546323006
Jul 17 03:26:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:25.008Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_oom_events_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"telegraf-security\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689546316714
Jul 17 03:26:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:25.235Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"pcap-tsgx12\", asset_id=\"117\", condition=\"Progressing\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"configmap-reloader\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"true\"}" timestamp=1689546311122
Jul 17 03:26:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:25.262Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_labels\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_endpoint=\"prometheus-kube-state-metrics\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546311122
Jul 17 03:26:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:25.537Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_metadata_generation\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"prometheus-pushgateway-bd476bcf9\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546320068
Jul 17 03:26:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:25.666Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_multicast_packets_transmitted_total\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_2\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546314790
Jul 17 03:26:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:25.692Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-pod17a39b41_f716_4eab_a979_0f0f67f99785.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"slice\"}" timestamp=1689546318416
Jul 17 03:26:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:25.781Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"kube-state-metrics\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Read\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546318907
Jul 17 03:26:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:25.807Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_limit_bytes\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"monit-stream\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546323419
Jul 17 03:26:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:25.836Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"java_lang_GarbageCollector_CollectionTime\", asset=\"pcap-dellr740-dt003\", asset_id=\"35\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-Yarn-pcap-dellr740-dt003-NodeManager\", endpoint_id=\"361\", module=\"DC-Yarn\", module_id=\"1\", name=\"PS Scavenge\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546325525
Jul 17 03:26:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:26.027Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-a675f3d38358a7aaabb7854e5e18bf454618c3432aec1414f1ea31c97a2d9987.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"scope\"}" timestamp=1689546311102
Jul 17 03:26:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:26.153Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_cache\", asset=\"twa-tsgx09\", asset_id=\"107\", container=\"kube-multus\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546318999
Jul 17 03:26:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:26.287Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"tmpfs\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/var/lib/kubelet/pods/8133a051-391d-44a4-bc3f-8749cb1b67a5/volumes/kubernetes.io~projected/kube-api-access-khvz6\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546320160
Jul 17 03:26:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:26.295Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-ask-password-wall.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"simple\", type=\"service\"}" timestamp=1689546318416
Jul 17 03:26:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:26.295Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_dormant\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens3f0np0\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546320160
Jul 17 03:26:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:26.341Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"35\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546315810
Jul 17 03:26:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:26.393Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_compressed_total\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vetha8a8b4ed\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546315810
Jul 17 03:26:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:26.498Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546315810
Jul 17 03:26:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:26.522Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"58\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546320160
Jul 17 03:26:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:26.536Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_packets_total\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth02cfbf6a\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546315810
Jul 17 03:26:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:26.540Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_unicast_packets_transmitted_total\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_2\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546315810
Jul 17 03:26:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:26.587Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"slice\"}" timestamp=1689546313907
Jul 17 03:26:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:26.726Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_timer_last_trigger_seconds\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sysstat-collect.timer\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689546313907
Jul 17 03:26:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:26.950Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_packets_total\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens7f1np1\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546315810
Jul 17 03:26:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:26.952Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_NodeManager_PublishAvgTime\", asset=\"pcap-dellr740-dt003\", asset_id=\"35\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-Yarn-pcap-dellr740-dt003-NodeManager\", endpoint_id=\"361\", module=\"DC-Yarn\", module_id=\"1\", name=\"MetricsSystem\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Stats\"}" timestamp=1689546325525
Jul 17 03:26:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:26.990Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"119\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546320160
Jul 17 03:26:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:27.026Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"9\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546320160
Jul 17 03:26:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:27.031Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_writes_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"coredns\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546312466
Jul 17 03:26:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:27.035Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"monit-stream\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Total\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546310242
Jul 17 03:26:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:27.081Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_annotations\", asset=\"pcap-tsgx12\", asset_id=\"117\", configmap=\"sapp-tsg-traffic-engine-numa-0\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546311122
Jul 17 03:26:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:27.089Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_cpu_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-podab6b1591_cfaa_4964_8fcc_a861d0862792.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"slice\"}" timestamp=1689546311102
Jul 17 03:26:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:27.107Z caller=dedupe.go:112 component=remote level=error remote_name=52db6f url=http://10.10.10.159:10090/nz-agent/prometheus/proxy/api/v1/write msg="non-recoverable error" count=1000 exemplarCount=0 err="server returned HTTP status 400 : out of order sample"
Jul 17 03:26:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:27.125Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"tmpfs\", name=\"var-lib-kubelet-pods-cbd312de\\\\x2d6144\\\\x2d4424\\\\x2d8813\\\\x2dfba2b3b16e9b-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2dkm9bh.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689546311102
Jul 17 03:26:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:27.136Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"nfsdcld.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"service\"}" timestamp=1689546311102
Jul 17 03:26:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:27.538Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"13\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546315810
Jul 17 03:26:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:27.579Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"twa-tsgx09\", asset_id=\"107\", container=\"dign-server-dns\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"cadvisor\", failure_type=\"pgmajfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"container\"}" timestamp=1689546316987
Jul 17 03:26:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:27.580Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"kubernetes-dashboard\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546318744
Jul 17 03:26:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:27.606Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_gc_sys_bytes\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"ipmi-exporter\"}" timestamp=1689546321597
Jul 17 03:26:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:27.782Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_type\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.configmap-reloader.v1\", service=\"prometheus-kube-state-metrics\", type=\"helm.sh/release.v1\"}" timestamp=1689546315682
Jul 17 03:26:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:27.783Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"twa-tsgx09\", asset_id=\"107\", condition=\"unknown\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-server-6ff784d758-mtn8x\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"55c0a8e2-51a8-48bf-a3cc-51508b7d7254\"}" timestamp=1689546318744
Jul 17 03:26:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:27.817Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_mtu_bytes\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth75455655\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546322882
Jul 17 03:26:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:27.852Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", fstype=\"ext4\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/var/lib/kubelet/pods/48a4a0ca-42d3-4487-bc89-7615c4f62e66/volume-subpaths/config-volume/telegraf-security/2\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546315810
Jul 17 03:26:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:27.885Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"100\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546322882
Jul 17 03:26:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:27.956Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_packets_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth2e13d535\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546322882
Jul 17 03:26:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:27.997Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"29\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546322882
Jul 17 03:26:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:28.008Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_up_changes_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens2f1np1\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546322882
Jul 17 03:26:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:28.009Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"mrzcpd_hugepages_setup.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"service\"}" timestamp=1689546314114
Jul 17 03:26:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:28.059Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_address_assign_type\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens3f1np1\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546315810
Jul 17 03:26:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:28.059Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link_mode\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"virtio_dign_c\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546322882
Jul 17 03:26:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:28.728Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"msh-tsgx10\", asset_id=\"69\", condition=\"Progressing\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"kubernetes-dashboard\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"true\"}" timestamp=1689546315682
Jul 17 03:26:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:28.757Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_rss\", asset=\"twa-tsgx09\", asset_id=\"107\", container=\"telegraf-security\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689546308144
Jul 17 03:26:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:28.790Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_ready_replicas\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"configmap-reloader-6c55bbc49f\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546318744
Jul 17 03:26:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:28.799Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2dpartuuid-c21ebe3b\\\\x2d6578\\\\x2d4fee\\\\x2da950\\\\x2de26fae2ff88e.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689546314114
Jul 17 03:26:29 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:29.098Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_annotations\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"kubernetes-dashboard\", uid=\"e77acf5f-c29e-4b1a-8ef9-268eb4abff25\"}" timestamp=1689546318744
Jul 17 03:26:29 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:29.163Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_storageclass_created\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", storageclass=\"local-path\"}" timestamp=1689546318744
Jul 17 03:26:29 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:29.321Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"loki_bytes_per_line_bucket\", agent_name=\"msh-kvm-nz001\", application=\"loki\", datacenter=\"MSH\", instance=\"127.0.0.1:13100\", le=\"512.0\"}" timestamp=1689546328327
Jul 17 03:26:30 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:30.057Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_colls_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens2f0np0\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546322882
Jul 17 03:26:30 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:30.254Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"dign-server-dns\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", failure_type=\"pgfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"hierarchy\"}" timestamp=1689546316730
Jul 17 03:26:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:31.303Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_fifo_total\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth2cff15d8\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546320939
Jul 17 03:26:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:31.347Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx05\", asset_id=\"67\", container=\"dign-server-dns\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Total\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546301929
Jul 17 03:26:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:31.674Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"dign-server-p4cvw\", project=\"TSG-OS\", project_id=\"7\", reason=\"NodeAffinity\", service=\"prometheus-kube-state-metrics\", uid=\"b3068a5c-f8e2-4c2d-89be-7648e87ab2c1\"}" timestamp=1689546330436
Jul 17 03:26:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:31.718Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_scrape_collector_success\", asset=\"msh-tsgx11\", asset_id=\"111\", collector=\"diskstats\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546320791
Jul 17 03:26:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:31.728Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_metadata_generation\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-0-firewall-6c48589865\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546330436
Jul 17 03:26:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:31.818Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_spec_completions\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", job_name=\"helm-install-traefik\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546324068
Jul 17 03:26:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:31.832Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_size_bytes\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"shm\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/8d39a9517b918329c7f81c70c48473c00048fa57270b260f86a901ffc615c9d2/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546320939
Jul 17 03:26:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:31.850Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_start_time_seconds\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"telegraf\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689546322009
Jul 17 03:26:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:31.883Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-virtual-net-veth2cff15d8.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689546321015
Jul 17 03:26:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:31.965Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"57\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546320939
Jul 17 03:26:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:31.980Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"40\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546320939
Jul 17 03:26:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:32.106Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_packets_total\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens4f2np2\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546320939
Jul 17 03:26:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:32.180Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_cpu_seconds_total\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-pod1d72bbea_870a_432c_b71c_26fca67b7d6c.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"slice\"}" timestamp=1689546321015
Jul 17 03:26:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:32.186Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"msh-tsgx05\", asset_id=\"67\", condition=\"true\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"cm-policy-local-cache-redis-master-69967b5879-rj8lj\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"adf25328-be5b-4a8f-99a7-ca39872df08e\"}" timestamp=1689546324068
Jul 17 03:26:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:32.195Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2did-scsi\\\\x2dSDELL_PERC_H345_Adp_00d652a8ad7b182c2c00f5d485f0a7ce\\\\x2dpart5.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689546321015
Jul 17 03:26:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:32.286Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_created\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"chart-values-traefik\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546324068
Jul 17 03:26:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:32.410Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"72\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546320791
Jul 17 03:26:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:32.520Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_cpu_sys_children_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546329260
Jul 17 03:26:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:32.695Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"106\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546320605
Jul 17 03:26:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:32.736Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"24\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546319093
Jul 17 03:26:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:32.760Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_bytes_total\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth1618597b\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546319093
Jul 17 03:26:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:32.916Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas_ready\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"configmap-reloader\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546319276
Jul 17 03:26:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:32.963Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"msh-tsgx11\", asset_id=\"111\", condition=\"Available\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"traefik\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"false\"}" timestamp=1689546319276
Jul 17 03:26:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:33.058Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546319093
Jul 17 03:26:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:33.299Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_flags\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"lo\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546319093
Jul 17 03:26:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:33.514Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_hwmon_sensor_label\", asset=\"twa-tsgx03\", asset_id=\"95\", chip=\"pci0000:00_0000:00:19_3\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", label=\"tctl\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", sensor=\"temp1\", service=\"prometheus-node-exporter\"}" timestamp=1689546319093
Jul 17 03:26:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:33.569Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_info\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_endpoint=\"cm-policy-local-cache-redis-headless\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546330436
Jul 17 03:26:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:33.572Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_metadata_resource_version\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"chart-values-tsg-traffic-engine-numa-0\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546324068
Jul 17 03:26:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:33.622Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-5cccc2781248b9b56b6d4fa749ec3224f3942836c778396419467980f2edcce6.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"scope\"}" timestamp=1689546321015
Jul 17 03:26:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:33.623Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"19\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546320939
Jul 17 03:26:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:33.641Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_info\", address=\"bc:97:e1:dd:99:d2\", asset=\"msh-tsgx05\", asset_id=\"67\", broadcast=\"ff:ff:ff:ff:ff:ff\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens4f2np2\", duplex=\"unknown\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operstate=\"down\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546320939
Jul 17 03:26:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:33.668Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"dign-client\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Discard\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546303305
Jul 17 03:26:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:33.726Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_start_time_seconds\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"kube-state-metrics\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546322009
Jul 17 03:26:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:33.734Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_writes_total\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda2\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546314223
Jul 17 03:26:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:33.736Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_replicas\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-0-firewall-5bc67584dc\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546319894
Jul 17 03:26:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:33.743Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"81\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546319093
Jul 17 03:26:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:33.811Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"13\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546320605
Jul 17 03:26:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:34.208Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sshd-keygen.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"target\"}" timestamp=1689546322644
Jul 17 03:26:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:34.492Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-eeb2a182bfdf0a771b05f50b41da759fe6d681b20968bb0b896a15806a59d845.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"scope\"}" timestamp=1689546326907
Jul 17 03:26:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:34.507Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-burstable-pod37e5fd6b_7636_4fb8_b761_e764c3267172.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"slice\"}" timestamp=1689546322644
Jul 17 03:26:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:34.704Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_fully_labeled_replicas\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-0-firewall-5bc67584dc\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546319894
Jul 17 03:26:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:34.870Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", fstype=\"ext4\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/data/var/lib/kubelet/pods/9982afd0-5736-4dce-b764-2f7b0c30831a/volume-subpaths/config-volume/telegraf-security/1\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546320605
Jul 17 03:26:35 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:35.088Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"30\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546320605
Jul 17 03:26:35 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:35.267Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_info\", asset=\"twa-tsgx05\", asset_id=\"99\", container=\"ipmi-exporter\", container_id=\"containerd://ccd21f147f54afc9f85b62ad9f1f20645fae02f8310f1ea89a64bcab9c1655a8\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", image=\"docker.io/pnnlmiscscripts/ipmi-exporter:1.2.0-9\", image_id=\"sha256:69b7e13cc9877c8676efbecca48c73b88b2a0c442d08f05421bebb6b7df404e3\", image_spec=\"docker.io/pnnlmiscscripts/ipmi-exporter:1.2.0-9\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"ipmi-exporter-6f8464dd94-lvhsz\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"a0e5cbc1-daaa-4b9e-8b09-4f9b6a34498e\"}" timestamp=1689546319894
Jul 17 03:26:37 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:37.792Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kafka_server_BrokerTopicMetrics_OneMinuteRate\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-Kafka-twa-dellr740-dt001\", endpoint_id=\"105\", module=\"DC-Kafka\", module_id=\"5\", name=\"ProduceMessageConversionsPerSec\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", topic=\"SESSION-RECORD-COMPLETED\"}" timestamp=1689546337482
Jul 17 03:26:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:40.507Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"74\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546331647
Jul 17 03:26:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:40.572Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_info\", asset=\"twa-tsgx04\", asset_id=\"97\", cluster_ip=\"10.43.200.62\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"ipmi-exporter\", uid=\"874f7f4f-9c48-45d3-bd39-9bdbf8e23915\"}" timestamp=1689546340034
Jul 17 03:26:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:40.590Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"32\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546331647
Jul 17 03:26:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:40.659Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"11\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546331647
Jul 17 03:26:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:40.865Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"twa-tsgx04\", asset_id=\"97\", container=\"dign-server-web\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"stopped\"}" timestamp=1689546319191
Jul 17 03:26:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:40.902Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_spec_paused\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"metrics-server\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546340034
Jul 17 03:26:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:40.928Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_spec_strategy_rollingupdate_max_surge\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"dashboard-metrics-scraper\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546340034
Jul 17 03:26:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:40.953Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_device_id\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens6f1np1\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546331647
Jul 17 03:26:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:41.043Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db142\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546333827
Jul 17 03:26:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:41.072Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_cache\", asset=\"twa-tsgx04\", asset_id=\"97\", container=\"kubernetes-dashboard\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546328431
Jul 17 03:26:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:41.154Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_mod_info\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"golang.org/x/sys\", nz_agent_id=\"3\", program=\"systemd_exporter\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", version=\"v0.0.0-20200321134203-328b4cd54aae\"}" timestamp=1689546335764
Jul 17 03:26:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:41.218Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"ext4\", name=\"var-lib-kubelet-pods-9ad11045\\\\x2de0bd\\\\x2d4481\\\\x2d9c2d\\\\x2d4c81a67475ee-volume\\\\x2dsubpaths-config\\\\x2dvolume-telegraf\\\\x2dsecurity-2.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689546335764
Jul 17 03:26:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:41.246Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-2133adaf\\\\x2dcc90\\\\x2d44bb\\\\x2d9cd8\\\\x2de180b9c1e90b-volume\\\\x2dsubpaths-telegraf-telegraf-0.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689546335764
Jul 17 03:26:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:41.584Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_namespace_annotations\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546340034
Jul 17 03:26:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:41.607Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"100\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546331647
Jul 17 03:26:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:41.739Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_maxStoreFileAge\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", namespace=\"system\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"b0132e934b40123d0abf0a3c94a74975\", table=\"bucket_meta\", type=\"data\"}" timestamp=1689546339758
Jul 17 03:26:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:41.960Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_storeCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"default\", project=\"OLAP\", project_id=\"3\", region=\"4fc005329c2eab495e1dfadd3fe1163f\", table=\"index_partfile_default\", type=\"index\"}" timestamp=1689546339758
Jul 17 03:26:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:41.976Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"40\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546331647
Jul 17 03:26:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:41.982Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"twa-tsgx04\", asset_id=\"97\", container=\"dign-server-web\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"cadvisor\", failure_type=\"pgmajfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"container\"}" timestamp=1689546319191
Jul 17 03:26:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:41.995Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_lastMajorCompactionAge\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"knowledge_base_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"ad22fcb50f84942952242dc36cd4fbaa\", table=\"index_filename_knowledge_base_hos_bucket\", type=\"index\"}" timestamp=1689546339758
Jul 17 03:26:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:42.036Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_DeleteBatch_99th_percentile\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Server\"}" timestamp=1689546339758
Jul 17 03:26:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:42.100Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"java_lang_MemoryPool_PeakUsage_committed\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"Code Cache\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546339758
Jul 17 03:26:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:42.117Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"twa-tsgx04\", asset_id=\"97\", container=\"exporter-merger\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"cadvisor\", failure_type=\"pgmajfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"hierarchy\"}" timestamp=1689546311488
Jul 17 03:26:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:42.131Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_info\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", job_name=\"helm-install-traefik-crd\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546340034
Jul 17 03:26:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:42.155Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_timex_status\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546331647
Jul 17 03:26:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:42.252Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_mapped_file\", asset=\"twa-tsgx04\", asset_id=\"97\", container=\"telegraf\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689546322517
Jul 17 03:26:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:42.265Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_appendCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"proxy_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"c01229b821e0a41e45bab953568a5e51\", table=\"index_partfile_proxy_hos_bucket\", type=\"index\"}" timestamp=1689546339758
Jul 17 03:26:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:42.709Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"rescue.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"idle\", type=\"service\"}" timestamp=1689546340671
Jul 17 03:26:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:43.039Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_labels\", asset=\"pcap-tsgx01\", asset_id=\"71\", configmap=\"tfe-tsg-traffic-engine-numa-0\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546335154
Jul 17 03:26:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:43.053Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_complete\", asset=\"pcap-tsgx01\", asset_id=\"71\", condition=\"true\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", job_name=\"helm-install-traefik-crd\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546335154
Jul 17 03:26:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:43.179Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"49\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546336242
Jul 17 03:26:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:43.343Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:00-0000:00:01.1-0000:01:00.1-infiniband-mlx5_1.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689546340671
Jul 17 03:26:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:43.359Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-rdma-devices-mlx5_1:1-umad.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689546340671
Jul 17 03:26:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:43.447Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-ac427b183d14c5c961a9825e68c5a40dabd24d8d77922e52d18c745e92e2aff7.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"scope\"}" timestamp=1689546340671
Jul 17 03:26:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:43.663Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:20-0000:20:03.1-0000:21:00.1-net-ens2f1np1.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689546340671
Jul 17 03:26:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:43.734Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens2f1np1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546336242
Jul 17 03:26:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:43.905Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-8a26b0c80d4e704c4f957fa48b9acd8e743177c1026a3e04589da13e26c9e30e.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"scope\"}" timestamp=1689546340671
Jul 17 03:26:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:49.198Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_maxStoreFileAge\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"361c4066fe82dcf3d9c18fdb1e80a7b4\", table=\"rtp_hos_bucket\", type=\"data\"}" timestamp=1689546347439
Jul 17 03:26:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:49.222Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_storeCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"assessment_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"43c96dd21aa1238d6db7226daea0110e\", table=\"index_filename_assessment_hos_bucket\", type=\"index\"}" timestamp=1689546347439
Jul 17 03:26:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:49.332Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_LOG_metric_tableReadQueryPerSecond_15min_rate\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546347439
Jul 17 03:26:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:49.336Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_1_metric_getTime_99_9th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546347439
Jul 17 03:26:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:49.382Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_194_2_metric_compactionOutputFileCount_75th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546347439
Jul 17 03:26:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:49.413Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_4_metric_getTime_99_9th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546347439
Jul 17 03:26:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:49.425Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_writeRequestCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"firewall_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"569c4ca2944511d5d885d0d8a8bc9016\", table=\"index_partfile_firewall_hos_bucket\", type=\"index\"}" timestamp=1689546347439
Jul 17 03:26:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:49.434Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_maxStoreFileAge\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"assessment_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"94a205daa3421c40aecd15d3f2e84511\", table=\"index_filename_assessment_hos_bucket\", type=\"index\"}" timestamp=1689546347439
Jul 17 03:26:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:49.470Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_memory_Bounce_bytes\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546341797
Jul 17 03:26:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:49.482Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_putCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"assessment_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"78cf5494be26ce7a79f8ec66b2777f96\", table=\"index_time_assessment_hos_bucket\", type=\"index\"}" timestamp=1689546347439
Jul 17 03:26:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:49.484Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_compactionsQueuedCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", namespace=\"dos\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"95cb8b9024dc19784dcfcbeb7b13278c\", table=\"ddos_traffic_baselines\", type=\"data\"}" timestamp=1689546347439
Jul 17 03:26:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:49.515Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_incrementCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"4821cc6d0e06e36fa88a4933ecc65476\", table=\"proxy_hos_bucket\", type=\"data\"}" timestamp=1689546347439
Jul 17 03:26:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:49.541Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_threads\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"prometheus-server-configmap-reload\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546335914
Jul 17 03:26:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:49.550Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_193_1_metric_majorCompactionInputFileCount_90th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546347439
Jul 17 03:26:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:49.553Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_info\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"chart-values-configmap-reloader\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546339668
Jul 17 03:26:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:49.607Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_193_1_metric_flushTime_95th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546347439
Jul 17 03:26:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:49.653Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_minStoreFileAge\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"1ea977c3db1b903435b4378976ab594e\", table=\"troubleshooting_hos_bucket\", type=\"data\"}" timestamp=1689546347439
Jul 17 03:26:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:49.741Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_cellsSizeCompactedFromMob\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Server\"}" timestamp=1689546347439
Jul 17 03:26:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:49.785Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_2_metric_flushTime_99th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546347439
Jul 17 03:26:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:49.854Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_fully_labeled_replicas\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-1-firewall-7f75fd649d\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546339668
Jul 17 03:26:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:49.886Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_hbase_table_namespace_metric_compactionInputSize_95th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546347439
Jul 17 03:26:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:49.899Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_194_3_metric_majorCompactionOutputFileCount_99_9th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546347439
Jul 17 03:26:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:49.946Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_resident_memory_bytes\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"auditd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689546346554
Jul 17 03:26:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:50.025Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:c0-0000:c0:01.1-0000:c1:00.0-infiniband_mad-umad8.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689546346554
Jul 17 03:26:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:50.082Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"dashboard-metrics-scraper\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Sync\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546328987
Jul 17 03:26:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:50.099Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2did-scsi\\\\x2d362cea7f08346c1002c270339b2b39591\\\\x2dpart3.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689546346554
Jul 17 03:26:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:50.283Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_writes_bytes_total\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689546328687
Jul 17 03:26:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:50.339Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"dashboard-metrics-scraper\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"sleeping\"}" timestamp=1689546328987
Jul 17 03:26:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:50.343Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx08\", asset_id=\"105\", container=\"configmap-reloader\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Discard\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546338330
Jul 17 03:26:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:50.358Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_daemonset_created\", asset=\"twa-tsgx10\", asset_id=\"109\", daemonset=\"svclb-traefik-577a35d6\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546339668
Jul 17 03:26:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:50.361Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_load_average_10s\", asset=\"twa-tsgx08\", asset_id=\"105\", container=\"prometheus-pushgateway\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546341072
Jul 17 03:26:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:50.376Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethcddab70d\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546341797
Jul 17 03:26:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:50.388Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_fifo_total\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"eno34np1\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546341797
Jul 17 03:26:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:50.464Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_reads_merged_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"overlay_0-400\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546341162
Jul 17 03:26:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:50.527Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_writes_merged_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"overlay_0-402\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546341162
Jul 17 03:26:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:50.544Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_usage_bytes\", asset=\"twa-tsgx08\", asset_id=\"105\", container=\"prometheus-alertmanager-configmap-reload\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546332823
Jul 17 03:26:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:50.591Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_info\", asset=\"twa-tsgx10\", asset_id=\"109\", configmap=\"multus-cni-config\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546339668
Jul 17 03:26:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:50.603Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"twa-tsgx10\", asset_id=\"109\", condition=\"true\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-systemd-exporter-vq7tn\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"8760a158-2fa4-4332-99b5-4233fdf937ac\"}" timestamp=1689546339668
Jul 17 03:26:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:50.721Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"tmpfs\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/data/var/lib/kubelet/pods/e4e2672a-a045-406b-b912-a701b409de63/volumes/kubernetes.io~projected/kube-api-access-qxdsd\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546346709
Jul 17 03:26:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:50.861Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_cpu_period\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"telegraf-security\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689546339050
Jul 17 03:26:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:50.892Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_mod_info\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"github.com/alecthomas/units\", nz_agent_id=\"3\", program=\"systemd_exporter\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", version=\"v0.0.0-20190924025748-f65c72e2690d\"}" timestamp=1689546341057
Jul 17 03:26:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:50.894Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_complete\", asset=\"twa-tsgx08\", asset_id=\"105\", condition=\"unknown\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", job_name=\"helm-delete-tsg-traffic-engine-numa-1\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546341568
Jul 17 03:26:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:50.919Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_cpu_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"crond.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689546346554
Jul 17 03:26:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:50.926Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx10\", asset_id=\"109\", cpu=\"40\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546341797
Jul 17 03:26:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:50.945Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2did-scsi\\\\x2dSDELL_PERC_H345_Adp_009195b3b23903272c00c14683f0a7ce\\\\x2dpart5.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689546346554
Jul 17 03:26:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:50.965Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"52\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546346709
Jul 17 03:26:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:50.984Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"overlay\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-f4199c99b6b713260f774913b51c3c5fd4ec8d5d8cc8d7b11f17282194b40459-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689546346554
Jul 17 03:26:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:50.989Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_state_started\", asset=\"twa-tsgx08\", asset_id=\"105\", container=\"dign-client\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"dign-client-pv7jg\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"f9e681d8-8372-4551-8580-4a68b3fae8df\"}" timestamp=1689546341568
Jul 17 03:26:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:50.999Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-541bab6f5b69ec62a4bbbac81886b37f6ce5fdadef1cc6d1499fcf4f42a9b7f1-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689546341057
Jul 17 03:26:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:51.012Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_compressed_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethcf0ceaf9\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546346709
Jul 17 03:26:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:51.035Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_scaling_frequency_min_hertz\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"16\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546346709
Jul 17 03:26:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:51.063Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_constraint_errors_transmitted_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_10\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546346709
Jul 17 03:26:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:51.084Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"overlay\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-0d4c3656cc5657663213fda5f206ed876f8384877447e83a924f5232747ce852-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689546346554
Jul 17 03:26:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:51.126Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_annotations\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", job_name=\"helm-install-traefik\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546339668
Jul 17 03:26:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:51.156Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-server-6ff784d758-jgfzf\", project=\"TSG-OS\", project_id=\"7\", reason=\"NodeAffinity\", service=\"prometheus-kube-state-metrics\", uid=\"dc3f3ccc-197b-483a-887e-f7d934ff1ed5\"}" timestamp=1689546339668
Jul 17 03:26:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:51.272Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"prometheus-server-configmap-reload\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"running\"}" timestamp=1689546335914
Jul 17 03:26:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:51.296Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", failure_type=\"pgfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"hierarchy\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689546331423
Jul 17 03:26:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:51.523Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_bytes_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", interface=\"ens2f0np0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546342031
Jul 17 03:26:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:51.551Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_usage_bytes\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/data/var/lib/kubelet/pods/1617bf89-68ca-42e6-8a6e-a79f76bf18ce/volumes/kubernetes.io~projected/kube-api-access-hjpfx\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546341162
Jul 17 03:26:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:51.575Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"50\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546346709
Jul 17 03:26:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:51.594Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_daemonset_labels\", asset=\"twa-tsgx08\", asset_id=\"105\", daemonset=\"prometheus-node-exporter\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546341568
Jul 17 03:26:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:51.662Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_files\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda3\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", fstype=\"ext4\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/update\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546346709
Jul 17 03:26:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:51.684Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"72\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546346709
Jul 17 03:26:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:51.716Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", cpu=\"31\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-olap003\", endpoint_id=\"235\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"yarn_resourcemanager,zookeeper,nacos,kafka,spark,druid_data,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546349713
Jul 17 03:26:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:51.737Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_reads_merged_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/var/lib/kubelet/pods/59e0fcfb-c5b9-4bb0-aefc-a15fed75746e/volumes/kubernetes.io~projected/kube-api-access-2wnf5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546341162
Jul 17 03:26:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:51.764Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"twa-tsgx08\", asset_id=\"105\", condition=\"false\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-delete-tsg-traffic-engine-numa-1-7nr86\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"de8cc098-4a75-498b-9682-77e8086dba81\"}" timestamp=1689546341568
Jul 17 03:26:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:51.784Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_physical_state_id\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_11\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546346709
Jul 17 03:26:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:51.813Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_cpu_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"watchdog.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689546346554
Jul 17 03:26:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:51.852Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db2\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546347556
Jul 17 03:26:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:51.855Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"117\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546346709
Jul 17 03:26:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:52.499Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_netstat_UdpLite_InErrors\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546349337
Jul 17 03:26:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:52.517Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_bytes_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens4f1np1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546349337
Jul 17 03:26:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:52.530Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_info\", asset=\"pcap-tsgx11\", asset_id=\"115\", cluster_ip=\"10.43.177.13\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"metrics-server\", uid=\"21884a85-4dd5-4540-a672-83db9af8303c\"}" timestamp=1689546344478
Jul 17 03:26:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:52.565Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"tsg_master_unknown\", app_name=\"tsg_master\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"traffic-engine\", line_name=\"abort\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", table_name=\"matched_rule\", vsysId=\"1\"}" timestamp=1689546347223
Jul 17 03:26:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:52.584Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", cpu=\"110\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546349337
Jul 17 03:26:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:52.610Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_fifo_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens2f1np1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546349337
Jul 17 03:26:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:52.699Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"clevis-luks-askpass.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689546345496
Jul 17 03:26:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:52.743Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"java_lang_Threading_PeakThreadCount\", asset=\"msh-dellr740-dt003\", asset_id=\"9\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt003-Hmaster\", endpoint_id=\"385\", module=\"DC-HBase\", module_id=\"3\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546352402
Jul 17 03:26:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:52.992Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", container=\"prometheus-server\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Discard\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546337075
Jul 17 03:26:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:53.035Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_created\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", job_name=\"helm-install-tsg-traffic-engine-numa-0\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546344478
Jul 17 03:26:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:53.048Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Dup_Udp_Bit\", app_name=\"sapp4\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", vsysId=\"1\"}" timestamp=1689546342596
Jul 17 03:26:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:53.048Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_compressed_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth36d3ae80\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546349337
Jul 17 03:26:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:53.098Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_restarts_total\", asset=\"msh-tsgx08\", asset_id=\"65\", container=\"prometheus-alertmanager\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-alertmanager-6df7f457b6-hnptg\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"9d9081be-60cd-45fa-aaca-20a311bb1d77\"}" timestamp=1689546348950
Jul 17 03:26:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:53.124Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_reservation_limit_bytes\", asset=\"msh-tsgx08\", asset_id=\"65\", container=\"prometheus-alertmanager\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546337907
Jul 17 03:26:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:53.136Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-infiniband-umad4.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689546345496
Jul 17 03:26:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:53.144Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled_time\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"mrzcpd-exporter-5cf7f967cc-m79nw\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"2c4890df-bfb7-48e8-a274-494cc0f3691d\"}" timestamp=1689546348950
Jul 17 03:26:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:53.325Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-vethfaa6969e.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689546351179
Jul 17 03:26:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:53.340Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", phase=\"Unknown\", pod=\"dign-server-2hjx7\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"22030e9c-e81d-45d2-a48a-b2673ca30200\"}" timestamp=1689546344478
Jul 17 03:26:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:53.473Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_metadata_resource_version\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"chart-values-traefik\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546344478
Jul 17 03:26:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:53.555Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_created\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"kubernetes-dashboard-key-holder\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546348663
Jul 17 03:26:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:53.568Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"tmpfs\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546343391
Jul 17 03:26:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:53.799Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_files_free\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"shm\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/c385b4a5a900a117708c9f54577244bcf20366eb87264f699990f6363627e73e/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546349337
Jul 17 03:26:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:53.816Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_free_bytes\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"shm\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/a9011f6e3a4566a7feaeb28a5ed6eda14c5dedfa3db492ed9e31d50ef6fb4a69/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546349337
Jul 17 03:26:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:54.032Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_file_descriptors\", asset=\"pcap-tsgx07\", asset_id=\"85\", container=\"prometheus-alertmanager\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546345698
Jul 17 03:26:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:54.052Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-86626e2e6ce0421725182a77f38c2eb199e2d5f550a32b3c7152ca97b53d2aa1-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689546352135
Jul 17 03:26:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:54.083Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failcnt\", asset=\"pcap-tsgx07\", asset_id=\"85\", container=\"exporter\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546341822
Jul 17 03:26:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:54.194Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_hwmon_sensor_label\", asset=\"msh-tsgx08\", asset_id=\"65\", chip=\"pci0000:00_0000:00:18_3\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", label=\"tccd6\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", sensor=\"temp8\", service=\"prometheus-node-exporter\"}" timestamp=1689546348328
Jul 17 03:26:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:54.243Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_lookups_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546339409
Jul 17 03:26:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:54.313Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_id\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth70701f01\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546339409
Jul 17 03:26:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:54.459Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_namespace_status_phase\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-public\", nz_agent_id=\"3\", phase=\"Active\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546348950
Jul 17 03:26:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:54.615Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_system_seconds_total\", asset=\"msh-tsgx12\", asset_id=\"113\", container=\"telegraf-security\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689546331344
Jul 17 03:26:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:54.669Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-f794203ae3e58bd3f51061f9a94f306697aa89cc449c3c231fde93d7bba3d052.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"scope\"}" timestamp=1689546351179
Jul 17 03:26:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:54.749Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_queue_length\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"vethf702ab1f\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546343391
Jul 17 03:26:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:54.839Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_type\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.systemd-exporter.v1\", service=\"prometheus-kube-state-metrics\", type=\"helm.sh/release.v1\"}" timestamp=1689546348663
Jul 17 03:26:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:54.890Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"tmpfs\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/var/lib/kubelet/pods/e3fb9886-6958-4504-8536-21d26d231bc8/volumes/kubernetes.io~projected/kube-api-access-8zdwv\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546343391
Jul 17 03:26:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:54.913Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx08\", asset_id=\"65\", cpu=\"92\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546348328
Jul 17 03:26:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:54.923Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_timex_pps_jitter_seconds\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546348328
Jul 17 03:26:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:55.267Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"msh-tsgx12\", asset_id=\"113\", condition=\"false\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"kube-multus-ds-87zqt\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"8b2e9710-4fd8-4c6f-8e5f-798d330e80a6\"}" timestamp=1689546348663
Jul 17 03:26:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:55.294Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_replicas\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"cm-policy-local-cache-redis-master-69967b5879\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546348663
Jul 17 03:26:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:55.334Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_down_changes_total\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"virtio_dign_s\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546343391
Jul 17 03:26:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:55.661Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_bytes_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"cadvisor\", interface=\"cnibr0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546341667
Jul 17 03:26:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:56.244Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_dormant\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"eno33np0\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546345883
Jul 17 03:26:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:56.294Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", container=\"dign-server-web\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Discard\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546350072
Jul 17 03:26:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:56.300Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"81\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546345883
Jul 17 03:26:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:56.319Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"49\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546345883
Jul 17 03:26:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:56.336Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dracut-pre-pivot.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"oneshot\", type=\"service\"}" timestamp=1689546343372
Jul 17 03:26:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:56.339Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_oom_events_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", container=\"configmap-reloader\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546344798
Jul 17 03:26:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:56.358Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"initrd-cleanup.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"service\"}" timestamp=1689546343372
Jul 17 03:26:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:56.490Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-aebe8d09\\\\x2d6d5d\\\\x2d4eda\\\\x2db90d\\\\x2d7be24e2da709-volume\\\\x2dsubpaths-config\\\\x2dvolume-telegraf-1.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689546343372
Jul 17 03:26:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:56.507Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-infiniband-umad1.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689546343372
Jul 17 03:26:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:56.630Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_info\", asset=\"pcap-tsgx03\", asset_id=\"73\", container=\"helm\", container_id=\"containerd://b3db6f763626da47d4bbb57f847a5358e23e198e57b26af9a923f43685f30073\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", image=\"docker.io/rancher/klipper-helm:v0.8.0-build20230510\", image_id=\"sha256:6f42df210d7faa0b6eb56f5f28bb061930c7834af1e7dd4b3586108ec6b1485b\", image_spec=\"rancher/klipper-helm:v0.8.0-build20230510\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-configmap-reloader-7csjj\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"1db50d06-c9a6-4b94-9812-73dffa7753e8\"}" timestamp=1689546353389
Jul 17 03:26:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:56.703Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_ports\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_endpoint=\"kube-dns\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", port_name=\"dns-tcp\", port_number=\"53\", port_protocol=\"TCP\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546353389
Jul 17 03:26:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:56.720Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_node_status_condition\", asset=\"pcap-tsgx03\", asset_id=\"73\", condition=\"PIDPressure\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", node=\"localhost\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"true\"}" timestamp=1689546353389
Jul 17 03:26:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:56.948Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_ips\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", ip=\"10.42.0.122\", ip_family=\"4\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", pod=\"tsg-traffic-engine-numa-1-firewall-5474c977-h4qcz\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"5b5ea4ec-9b19-421d-88cb-9ffacec87f13\"}" timestamp=1689546353389
Jul 17 03:26:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:57.057Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546345883
Jul 17 03:26:58 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:58.011Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_created\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-systemd-exporter-pw27n\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"6e4c6271-0705-4680-9a24-73bceb8107af\"}" timestamp=1689546353389
Jul 17 03:26:58 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:58.043Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"pcap-tsgx03\", asset_id=\"73\", condition=\"true\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"cm-policy-local-cache-redis-master-69967b5879-sjdjw\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"5c522fdc-927c-4897-9ae6-09e4fb08b6fe\"}" timestamp=1689546353389
Jul 17 03:26:58 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:58.083Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link_mode\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens2f0np0\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546345883
Jul 17 03:26:58 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:58.089Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_processes\", asset=\"pcap-tsgx03\", asset_id=\"73\", container=\"dign-server-web\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546350072
Jul 17 03:26:58 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:58.280Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_sector_reads_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda2\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546349930
Jul 17 03:26:58 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:58.492Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"tmpfs\", endpoint=\"OLAP-Node-Exporter-pcap-dellr740-dt001\", endpoint_id=\"251\", fstype=\"tmpfs\", module=\"OLAP-Node-Exporter\", module_id=\"51\", mountpoint=\"/run\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546357557
Jul 17 03:26:58 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:58.616Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", cpu=\"47\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"OLAP-Node-Exporter-pcap-dellr740-dt001\", endpoint_id=\"251\", mode=\"idle\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546357557
Jul 17 03:26:58 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:58.729Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-dellr740-dt003\", asset_id=\"9\", cpu=\"9\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-dt003\", endpoint_id=\"229\", mode=\"nice\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"hbase_master,clickhouse_data,kafka,gohangout,zookeeper,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546357407
Jul 17 03:26:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:59.102Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"OLAP-Node-Exporter-pcap-dellr740-dt001\", endpoint_id=\"251\", module=\"OLAP-Node-Exporter\", module_id=\"51\", name=\"17\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\", type=\"Processor\"}" timestamp=1689546357557
Jul 17 03:26:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:59.154Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-dellr740-dt003\", asset_id=\"9\", cpu=\"44\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-dt003\", endpoint_id=\"229\", mode=\"system\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"hbase_master,clickhouse_data,kafka,gohangout,zookeeper,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546357407
Jul 17 03:26:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:59.670Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_speed_bytes\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth55faa9d3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546349083
Jul 17 03:26:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:59.762Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_complete\", asset=\"msh-tsgx01\", asset_id=\"51\", condition=\"false\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", job_name=\"helm-install-mrzcpd-exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546347425
Jul 17 03:26:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:59.854Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"11\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546349083
Jul 17 03:26:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:26:59.911Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_receive_switch_relay_errors_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_4\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546349083
Jul 17 03:27:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:00.221Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx01\", asset_id=\"51\", container=\"metrics\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Read\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546342195
Jul 17 03:27:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:00.275Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_cpu_period\", asset=\"msh-tsgx01\", asset_id=\"51\", container=\"telegraf-security\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689546354593
Jul 17 03:27:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:01.448Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_restarts_total\", asset=\"msh-tsgx01\", asset_id=\"51\", container=\"exporter\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"systemd-exporter-7f549d9485-qmd8f\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"45b995ae-405c-4e5d-b21a-3830d4e78e4d\"}" timestamp=1689546347425
Jul 17 03:27:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:01.577Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", phase=\"Failed\", pod=\"tsg-traffic-engine-numa-1-firewall-6c6f4bc877-xxxz9\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"b3efc35a-4844-47d9-82c9-7d335d0d4eaa\"}" timestamp=1689546347425
Jul 17 03:27:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:01.594Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"msh-tsgx01\", asset_id=\"51\", condition=\"unknown\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-kube-state-metrics-6b768c7bf8-fnk7m\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"75a1cd69-272c-4ee3-afeb-f27cbbc02353\"}" timestamp=1689546347425
Jul 17 03:27:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:01.598Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_packets_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens3f1np1\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546349083
Jul 17 03:27:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:01.615Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_labels\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", uid=\"64343e3d-b361-4200-a30a-dbeb74f9ea55\"}" timestamp=1689546347425
Jul 17 03:27:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:01.627Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"import-state.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689546344879
Jul 17 03:27:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:01.633Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"96\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546349083
Jul 17 03:27:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:02.480Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_packets_total\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", interface=\"ens4f1np1\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546350854
Jul 17 03:27:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:02.585Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_rss\", asset=\"msh-tsgx02\", asset_id=\"63\", container=\"coredns\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546333565
Jul 17 03:27:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:02.727Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_flags\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens2f0np0\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546358318
Jul 17 03:27:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:02.839Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_packets_dropped_total\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", interface=\"virtio_dign_c\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546350854
Jul 17 03:27:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:02.852Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_info\", asset=\"msh-tsgx02\", asset_id=\"63\", cluster_ip=\"10.43.122.41\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"dashboard-metrics-scraper\", uid=\"5820ca97-9db2-486f-86c1-89b7ea112b86\"}" timestamp=1689546351412
Jul 17 03:27:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:02.854Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"msh-tsgx02\", asset_id=\"63\", container=\"firewall\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", failure_type=\"pgmajfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"hierarchy\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689546347531
Jul 17 03:27:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:02.898Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_packets_total\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", interface=\"eno1\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546350854
Jul 17 03:27:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:02.942Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"initrd-parse-etc.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"service\"}" timestamp=1689546348447
Jul 17 03:27:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:03.312Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_status_start_time\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", job_name=\"helm-install-configmap-reloader\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546351412
Jul 17 03:27:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:03.708Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"net_conntrack_dialer_conn_failed_total\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", dialer_name=\"203\", instance=\"10.10.20.159:10090\", reason=\"resolution\", type=\"global\"}" timestamp=1689546362989
Jul 17 03:27:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:03.790Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_discards_transmitted_total\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_1\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546358318
Jul 17 03:27:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:04.270Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_gc_duration_seconds_count\", asset=\"msh-dellr740-olap005\", asset_id=\"19\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Spark-msh-dellr740-olap005\", endpoint_id=\"17\", module=\"NC-Spark\", module_id=\"47\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546363391
Jul 17 03:27:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:04.323Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"net_conntrack_dialer_conn_attempted_total\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", dialer_name=\"25\", instance=\"10.10.20.159:10090\", type=\"global\"}" timestamp=1689546362989
Jul 17 03:27:05 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:05.510Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-dellr740-olap005\", asset_id=\"19\", cpu=\"29\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Spark-msh-dellr740-olap005\", endpoint_id=\"17\", mode=\"nice\", module=\"NC-Spark\", module_id=\"47\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546363391
Jul 17 03:27:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:07.614Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", cpu=\"24\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Spark-msh-dellr740-olap003\", endpoint_id=\"13\", module=\"NC-Spark\", module_id=\"47\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546366091
Jul 17 03:27:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:07.683Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_labels\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"metrics-server-54dc485875-l4kw8\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"e96787f3-8c20-4c5e-a395-ec874ca09072\"}" timestamp=1689546361227
Jul 17 03:27:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:07.748Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"126\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546363910
Jul 17 03:27:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:07.775Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cpu=\"86\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546363910
Jul 17 03:27:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:07.801Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link_mode\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens7f0np0\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546363910
Jul 17 03:27:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:08.104Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_packets_dropped_total\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"cadvisor\", interface=\"ens8f1np1\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546362048
Jul 17 03:27:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:08.122Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_spec_replicas\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"coredns-6d8dc697\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546361227
Jul 17 03:27:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:08.143Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"overlay\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-9b07bfd510c4a8a6d3044d5a96fdba1ac94bb7d3878a68b4f6fe473de287479b-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689546353582
Jul 17 03:27:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:08.187Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"twa-tsgx01\", asset_id=\"91\", condition=\"Available\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"metrics-server\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"unknown\"}" timestamp=1689546361227
Jul 17 03:27:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:08.329Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-7ce93833\\\\x2d4c85\\\\x2d4a59\\\\x2dbe25\\\\x2d1ba1b6f91e91-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2dx5dkl.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689546353582
Jul 17 03:27:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:08.410Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"scrape_samples_post_metric_relabeling\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_instance=\"10.42.0.153:9005\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"mrzcpd-exporter\"}" timestamp=1689546358833
Jul 17 03:27:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:08.540Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_colls_total\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens1f1np1\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546363910
Jul 17 03:27:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:08.562Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cpu=\"112\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546363910
Jul 17 03:27:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:08.568Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", pod=\"tsg-traffic-engine-numa-0-firewall-584cc6bdff-5rl2w\", project=\"TSG-OS\", project_id=\"7\", reason=\"UnexpectedAdmissionError\", service=\"prometheus-kube-state-metrics\", uid=\"cd272aa8-aacf-4f0d-971f-5dbf27e21826\"}" timestamp=1689546361227
Jul 17 03:27:09 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:09.677Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_lastMajorCompactionAge\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"proxy_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"2ea4c2e074dbea4bc82df3819be3b6ce\", table=\"index_time_proxy_hos_bucket\", type=\"index\"}" timestamp=1689546368804
Jul 17 03:27:09 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:09.713Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_compactionsFailedCount\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"a3eb769a8c2d325e0a97838f6786a569\", table=\"session_record_hos_bucket\", type=\"data\"}" timestamp=1689546368804
Jul 17 03:27:09 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:09.903Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_getCount\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"knowledge_base_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"ebc2debc1fb19c5df58bdfbbd410b63e\", table=\"index_time_knowledge_base_hos_bucket\", type=\"index\"}" timestamp=1689546368804
Jul 17 03:27:09 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:09.924Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_compactionsQueuedCount\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"41801e02474f23036d0c22fbb9f5902e\", table=\"session_record_hos_bucket\", type=\"data\"}" timestamp=1689546368804
Jul 17 03:27:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:10.104Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_numReferenceFiles\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"proxy_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"e82d9462faab70c1d607d81a60d135ec\", table=\"index_filename_proxy_hos_bucket\", type=\"index\"}" timestamp=1689546368804
Jul 17 03:27:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:10.142Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_writeRequestCount\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"default\", project=\"OLAP\", project_id=\"3\", region=\"61ecde15b0197c24043c6c1d3fa7957d\", table=\"index_time_default\", type=\"index\"}" timestamp=1689546368804
Jul 17 03:27:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:10.167Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_scanCount\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"rtp_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"da4d07ac6e142d510f271e8e5a276908\", table=\"index_filename_rtp_hos_bucket\", type=\"index\"}" timestamp=1689546368804
Jul 17 03:27:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:10.751Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_deleteCount\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"699a9b949da87530250d91a08d1c77d6\", table=\"rtp_hos_bucket\", type=\"data\"}" timestamp=1689546368804
Jul 17 03:27:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:11.028Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_flushesQueuedCount\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"rtp_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"1e962c0c5c99780cbb2864391a86c3e8\", table=\"index_time_rtp_hos_bucket\", type=\"index\"}" timestamp=1689546368804
Jul 17 03:27:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:11.366Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_3_metric_flushMemstoreSize_95th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546370147
Jul 17 03:27:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:11.383Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_readRequestCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"troubleshooting_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"8b6b702758ed0122cff87267ae423044\", table=\"index_time_troubleshooting_hos_bucket\", type=\"index\"}" timestamp=1689546370147
Jul 17 03:27:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:11.441Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_194_4_metric_appendTime_90th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546370147
Jul 17 03:27:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:11.449Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_appendCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"session_record_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"6dacfb7337a3da49e28f46ddfef67879\", table=\"index_partfile_session_record_hos_bucket\", type=\"index\"}" timestamp=1689546370147
Jul 17 03:27:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:11.462Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_compactionsCompletedCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"assessment_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"c326970ff7d07c2f0727098ea5f6dedb\", table=\"index_time_assessment_hos_bucket\", type=\"index\"}" timestamp=1689546370147
Jul 17 03:27:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:11.516Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_filteredReadRequestCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"proxy_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"2097b60eaa4ffea308d53e366e4242fb\", table=\"index_filename_proxy_hos_bucket\", type=\"index\"}" timestamp=1689546370147
Jul 17 03:27:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:11.520Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_LOG_metric_getTime_99_9th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546370147
Jul 17 03:27:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:11.535Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_tsg_galaxy_table_gtpc_knowledge_base_metric_getTime_25th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546370147
Jul 17 03:27:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:11.627Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_4_metric_compactionInputFileCount_median\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546370147
Jul 17 03:27:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:11.648Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_ScanSize_99th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Server\"}" timestamp=1689546370147
Jul 17 03:27:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:12.230Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_tsg_table_report_result_metric_putBatchTime_98th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546370147
Jul 17 03:27:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:12.275Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_storeFileSize\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Server\"}" timestamp=1689546370147
Jul 17 03:27:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:12.285Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_PostIndexUpdateFailureTime_99_9th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"PhoenixIndexer\"}" timestamp=1689546370147
Jul 17 03:27:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:12.585Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_numFilesCompactedCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"assessment_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"8c5d4ec339493f729dc1ab388351d74a\", table=\"index_filename_assessment_hos_bucket\", type=\"index\"}" timestamp=1689546370147
Jul 17 03:27:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:12.599Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_194_4_metric_memStoreSize\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546370147
Jul 17 03:27:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:12.613Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_4_metric_scanTime_max\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546370147
Jul 17 03:27:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:12.629Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_193_1_metric_flushOutputSize_median\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546370147
Jul 17 03:27:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:12.770Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Sink_ageOfLastAppliedOp_99th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Replication\"}" timestamp=1689546370147
Jul 17 03:27:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:12.806Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_session_record_hos_bucket_metric_scanTime_num_ops\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546370147
Jul 17 03:27:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:12.885Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_readRequestCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"troubleshooting_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"c925b6b3313af58468f0ac47618c47d4\", table=\"index_partfile_troubleshooting_hos_bucket\", type=\"index\"}" timestamp=1689546370147
Jul 17 03:27:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:12.903Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_4_metric_scanSize_90th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546370147
Jul 17 03:27:14 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:14.209Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_free_bytes\", asset=\"twa-dellr740-dt002\", asset_id=\"39\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sdc1\", endpoint=\"OLAP-Node-Exporter-twa-dellr740-dt002\", endpoint_id=\"259\", fstype=\"xfs\", module=\"OLAP-Node-Exporter\", module_id=\"51\", mountpoint=\"/data\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546372056
Jul 17 03:27:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:15.320Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-eda00de6\\\\x2d45a4\\\\x2d4eaf\\\\x2d8b0e\\\\x2d75c0b9c1b061-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2dczwm7.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689546367046
Jul 17 03:27:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:15.542Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kafka_server_socket_server_metrics_request_rate\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-Kafka-pcap-dellr740-dt001\", endpoint_id=\"99\", listener=\"SASL_PLAINTEXT\", module=\"DC-Kafka\", module_id=\"5\", networkProcessor=\"0\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546374294
Jul 17 03:27:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:15.939Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_rss\", asset=\"twa-tsgx07\", asset_id=\"103\", container=\"exporter-merger\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546364834
Jul 17 03:27:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:15.953Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_created\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-0-firewall-5b47889f4f\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546368076
Jul 17 03:27:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:16.410Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_created\", asset=\"twa-tsgx07\", asset_id=\"103\", configmap=\"multus-cni-config\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546368076
Jul 17 03:27:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:16.783Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"rpcbind.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"target\"}" timestamp=1689546367046
Jul 17 03:27:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:19.866Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_annotations\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"mrzcpd-exporter-5cf7f967cc-dxl48\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"5f8b23e9-8830-4b0a-9d61-06b46edd96f0\"}" timestamp=1689546373949
Jul 17 03:27:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:20.067Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"16\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546377612
Jul 17 03:27:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:20.125Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx06\", asset_id=\"101\", container=\"firewall\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Async\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689546372364
Jul 17 03:27:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:20.204Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2did-wwn\\\\x2d0x62cea7f0848275002c26f81cd27e8046\\\\x2dpart4.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689546364356
Jul 17 03:27:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:20.251Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_init_container_status_ready\", asset=\"twa-tsgx06\", asset_id=\"101\", container=\"install-multus-binary\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"kube-multus-ds-gdfsd\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"6b35184e-3858-440f-ae09-849d005db403\"}" timestamp=1689546373949
Jul 17 03:27:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:20.258Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546377612
Jul 17 03:27:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:20.273Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_metadata_resource_version\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"k3s-serving\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546373949
Jul 17 03:27:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:20.342Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_created\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"dashboard-metrics-scraper\", uid=\"325dd97e-7163-4029-ab6c-2360cae9de0a\"}" timestamp=1689546373949
Jul 17 03:27:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:20.352Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_running\", asset=\"pcap-tsgx10\", asset_id=\"89\", container=\"prometheus-server-configmap-reload\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-server-6ff784d758-wkch9\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"c8175cdd-27fb-4976-a041-ad5b277f4e45\"}" timestamp=1689546373828
Jul 17 03:27:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:20.375Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_compressed_total\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethac193cfe\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546377612
Jul 17 03:27:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:20.382Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"28\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546368263
Jul 17 03:27:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:20.400Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"108\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546368263
Jul 17 03:27:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:20.420Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"lldpad.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689546364356
Jul 17 03:27:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:20.508Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", container=\"metrics-server\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"cadvisor\", failure_type=\"pgfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"hierarchy\"}" timestamp=1689546367645
Jul 17 03:27:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:20.529Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas_updated\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"prometheus-kube-state-metrics\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546373828
Jul 17 03:27:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:20.549Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"48\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546368263
Jul 17 03:27:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:20.682Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:00-0000:00:01.1-0000:01:00.1-net-ens1f1np1.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689546364356
Jul 17 03:27:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:20.736Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"var-lib-kubelet-pods-9145d365\\\\x2d24c7\\\\x2d4ff1\\\\x2da1ee\\\\x2d0a800b33502b-volume\\\\x2dsubpaths-config\\\\x2dvolume-telegraf\\\\x2dsecurity-1.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689546364356
Jul 17 03:27:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:20.914Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"120\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546368263
Jul 17 03:27:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:20.940Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"9\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546368263
Jul 17 03:27:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:20.962Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"rsyslog.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"service\"}" timestamp=1689546370657
Jul 17 03:27:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:21.031Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_resident_memory_bytes\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"mrzcpd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689546364356
Jul 17 03:27:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:21.074Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"overlay\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-47b9ddb51c1e4552dcd76f41eb889391e91c84ba4925ce58fe0e39c76bbe4b79-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689546364356
Jul 17 03:27:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:21.181Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"42\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546368263
Jul 17 03:27:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:21.348Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx06\", asset_id=\"101\", container=\"dashboard-metrics-scraper\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Sync\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546359863
Jul 17 03:27:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:21.364Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_created\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"ipmi-exporter\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546373949
Jul 17 03:27:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:21.449Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_created\", asset=\"twa-tsgx06\", asset_id=\"101\", configmap=\"kube-root-ca.crt\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546373949
Jul 17 03:27:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:21.467Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"initrd.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"target\"}" timestamp=1689546364356
Jul 17 03:27:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:21.561Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_labels\", asset=\"pcap-tsgx10\", asset_id=\"89\", configmap=\"chart-content-configmap-reloader\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546373828
Jul 17 03:27:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:21.574Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"kube-multus-ds-sqrb7\", project=\"TSG-OS\", project_id=\"7\", reason=\"Shutdown\", service=\"prometheus-kube-state-metrics\", uid=\"ee8afe42-b7fb-4840-a9ee-3d0e7212c356\"}" timestamp=1689546373828
Jul 17 03:27:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:21.575Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"60\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546377612
Jul 17 03:27:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:21.591Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_info\", address=\"5e:2e:76:cd:26:23\", asset=\"pcap-tsgx10\", asset_id=\"89\", broadcast=\"ff:ff:ff:ff:ff:ff\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth36380b9e\", duplex=\"full\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operstate=\"up\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546368263
Jul 17 03:27:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:21.594Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"17\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546377612
Jul 17 03:27:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:21.638Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"shm\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/9fbd3ff04dd48db4b39f55e3c9158c39d3277e82aad9856f80daed4ca7246221/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546368263
Jul 17 03:27:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:21.650Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_file_descriptors\", asset=\"pcap-tsgx10\", asset_id=\"89\", container=\"exporter\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546359170
Jul 17 03:27:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:21.654Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"phy_tx_bits_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", dev=\"ens1f0np0\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"mrzcpd-exporter\"}" timestamp=1689546374036
Jul 17 03:27:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:21.807Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-5e13c3a0d5f9065ec9ae00372c26ba8786248c86182de51d1ae7eb8dffed8552.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"scope\"}" timestamp=1689546370657
Jul 17 03:27:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:21.829Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"iscsi-shutdown.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"service\"}" timestamp=1689546370657
Jul 17 03:27:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:21.914Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_os_version\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", id=\"rocky\", id_like=\"rhel centos fedora\", module=\"TSG-OS\", module_id=\"77\", name=\"TSG-OS\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546368263
Jul 17 03:27:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:21.923Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"11\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546368263
Jul 17 03:27:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:21.943Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"57\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546368263
Jul 17 03:27:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:21.969Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_files_free\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda5\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", fstype=\"ext4\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/data\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546368263
Jul 17 03:27:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:21.971Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-veth0aa05a07.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689546370657
Jul 17 03:27:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:22.015Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_heap_sys_bytes\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546374840
Jul 17 03:27:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:22.018Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_slave_priority\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546374840
Jul 17 03:27:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:22.027Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"123\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546368263
Jul 17 03:27:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:22.043Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db26\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546374840
Jul 17 03:27:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:22.166Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled_time\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-ipmi-exporter-ntwmm\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"6d124c79-91fd-435c-a730-222ebe3b4194\"}" timestamp=1689546373828
Jul 17 03:27:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:23.122Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"java_lang_MemoryPool_UsageThreshold\", asset=\"msh-dellr740-dt003\", asset_id=\"9\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-Yarn-msh-dellr740-dt003-NodeManager\", endpoint_id=\"355\", module=\"DC-Yarn\", module_id=\"1\", name=\"Metaspace\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546382405
Jul 17 03:27:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:23.150Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"java_lang_GarbageCollector_LastGcInfo_memoryUsageBeforeGc_committed\", asset=\"msh-dellr740-dt003\", asset_id=\"9\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-Yarn-msh-dellr740-dt003-NodeManager\", endpoint_id=\"355\", key=\"PS Survivor Space\", module=\"DC-Yarn\", module_id=\"1\", name=\"PS MarkSweep\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546382405
Jul 17 03:27:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:23.919Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db40\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546377019
Jul 17 03:27:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:24.208Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_fifo_total\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"eno2\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546374790
Jul 17 03:27:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:24.221Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"msh-tsgx09\", asset_id=\"55\", cpu=\"126\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546374790
Jul 17 03:27:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:24.271Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"pcap-tsgx12\", asset_id=\"117\", condition=\"true\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"systemd-exporter-7f549d9485-m5dr9\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"0557366f-a93d-40e0-ba86-42e3b764cacf\"}" timestamp=1689546371122
Jul 17 03:27:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:24.291Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-89567228\\\\x2d6ae4\\\\x2d4ed5\\\\x2d8117\\\\x2d7627ccbebb6d-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2dbhmtx.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689546371102
Jul 17 03:27:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:24.389Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_disk_writes_completed_total\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"sda\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546374790
Jul 17 03:27:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:24.398Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db13\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546377019
Jul 17 03:27:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:24.399Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db63\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546377019
Jul 17 03:27:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:24.417Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"pcap-tsgx12\", asset_id=\"117\", condition=\"true\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"svclb-traefik-05ed3a27-xh5sz\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"af82e0e9-3f8b-42a5-852d-e91e08de8066\"}" timestamp=1689546371122
Jul 17 03:27:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:24.431Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-pod012398ff_fe58_4ee4_8731_8e61bf317bbf.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"slice\"}" timestamp=1689546378416
Jul 17 03:27:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:24.512Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"dign-server-dns\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Total\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546380897
Jul 17 03:27:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:24.521Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_address_assign_type\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"bond0\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546374790
Jul 17 03:27:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:24.535Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"prometheus-node-exporter\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"iowaiting\"}" timestamp=1689546380025
Jul 17 03:27:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:24.541Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_resident_memory_bytes\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"getty@tty1.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689546378416
Jul 17 03:27:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:24.578Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_annotations\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_endpoint=\"kube-dns\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546380068
Jul 17 03:27:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:24.600Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_init_container_status_terminated\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"install-multus-binary\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"kube-multus-ds-dsd65\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"e9990f8b-7450-47a8-867f-7387f0ac7e25\"}" timestamp=1689546371122
Jul 17 03:27:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:24.616Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"74\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546380160
Jul 17 03:27:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:24.666Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:00-0000:00:01.1-0000:01:00.0-infiniband_mad-umad0.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689546378416
Jul 17 03:27:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:24.676Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_data_transmitted_bytes_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"mlx5_11\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546380160
Jul 17 03:27:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:24.678Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-8a01e35366ef3b7059bf39b93a69eff62b691b0f988b1e3f74f126fc8f86d4b9.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"scope\"}" timestamp=1689546378416
Jul 17 03:27:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:24.743Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_swap\", asset=\"msh-tsgx09\", asset_id=\"55\", container=\"kube-state-metrics\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546377461
Jul 17 03:27:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:24.755Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx09\", asset_id=\"55\", cpu=\"112\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546374790
Jul 17 03:27:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:24.769Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db150\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546377019
Jul 17 03:27:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:24.788Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"scrape_samples_scraped\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_instance=\"10.42.0.154:9005\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"mrzcpd-exporter\"}" timestamp=1689546380889
Jul 17 03:27:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:24.815Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"scrape_series_added\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_instance=\"10.10.10.172:9100\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546380160
Jul 17 03:27:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:24.938Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx09\", asset_id=\"55\", container=\"lb-tcp-80\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Write\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546370543
Jul 17 03:27:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:24.952Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_metadata_resource_version\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"chart-values-tsg-traffic-engine-numa-0\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546371122
Jul 17 03:27:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:24.956Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_limit_bytes\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546375807
Jul 17 03:27:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:24.964Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"41\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546380160
Jul 17 03:27:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:24.974Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"msh-tsgx09\", asset_id=\"55\", container=\"redis\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"iowaiting\"}" timestamp=1689546377118
Jul 17 03:27:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:24.977Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"import-state.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689546371102
Jul 17 03:27:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:24.990Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx09\", asset_id=\"55\", cpu=\"14\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546374790
Jul 17 03:27:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:25.010Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_id\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens8f1np1\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546374790
Jul 17 03:27:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:25.051Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"61\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546380160
Jul 17 03:27:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:25.071Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_dormant\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens6f0np0\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546380160
Jul 17 03:27:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:25.100Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_metadata_resource_version\", asset=\"msh-tsgx09\", asset_id=\"55\", configmap=\"kube-root-ca.crt\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-public\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546380068
Jul 17 03:27:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:25.120Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx09\", asset_id=\"55\", cpu=\"95\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546374790
Jul 17 03:27:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:25.124Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_protocol_type\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"vethc0962f36\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546380160
Jul 17 03:27:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:25.139Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_link_downed_total\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_0\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546374790
Jul 17 03:27:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:25.368Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_scrape_collector_success\", asset=\"pcap-tsgx12\", asset_id=\"117\", collector=\"time\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546380160
Jul 17 03:27:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:26.015Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_colls_total\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth02cfbf6a\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546375810
Jul 17 03:27:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:26.037Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"overlay\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-6dfb472b5c6f4e10ab26aea947c493f86c4b6a1735b0b3576b03366de4b17cd5-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689546373907
Jul 17 03:27:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:26.040Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-22272132c2da31b9ce148bbaf64f852a794dc33f9c28a5d495b535792db4e5bd.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"scope\"}" timestamp=1689546373907
Jul 17 03:27:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:26.078Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"119\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546375810
Jul 17 03:27:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:26.095Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-33fe7fac75e88dc1d94e65f8829d89582ef08956e98c2ed977039b7c8701cee5.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"scope\"}" timestamp=1689546373907
Jul 17 03:27:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:26.137Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_usage_bytes\", asset=\"twa-tsgx09\", asset_id=\"107\", container=\"kube-multus\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546375486
Jul 17 03:27:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:26.156Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dracut-pre-mount.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"service\"}" timestamp=1689546373907
Jul 17 03:27:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:26.332Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:c0-0000:c0:01.1-0000:c1:00.0-infiniband-mlx5_8.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689546373907
Jul 17 03:27:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:26.388Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"66\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546375810
Jul 17 03:27:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:26.393Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_data_received_bytes_total\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_6\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546375810
Jul 17 03:27:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:26.423Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_up_changes_total\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens1f1np1\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546375810
Jul 17 03:27:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:26.447Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_packets_total\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens8f0np0\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546375810
Jul 17 03:27:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:26.448Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"tmpfs\", name=\"var-lib-kubelet-pods-2edb1c6e\\\\x2d4f58\\\\x2d4c31\\\\x2db8c5\\\\x2d28dfc8f91f4d-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2d54cpn.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689546373907
Jul 17 03:27:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:26.838Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"110\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546375810
Jul 17 03:27:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:26.938Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_protocol_type\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens8f1np1\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546375810
Jul 17 03:27:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:27.074Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_device_id\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth188af860\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546375810
Jul 17 03:27:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:27.094Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"27\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546375810
Jul 17 03:27:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:27.131Z caller=dedupe.go:112 component=remote level=error remote_name=52db6f url=http://10.10.10.159:10090/nz-agent/prometheus/proxy/api/v1/write msg="non-recoverable error" count=1000 exemplarCount=0 err="server returned HTTP status 400 : out of order sample"
Jul 17 03:27:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:27.227Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"phy_tx_error_total\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", dev=\"ens7f1np1\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"mrzcpd-exporter\"}" timestamp=1689546378295
Jul 17 03:27:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:27.261Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_id\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens4f3np3\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546375810
Jul 17 03:27:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:27.285Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db223\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546383549
Jul 17 03:27:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:27.311Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx09\", asset_id=\"107\", container=\"dign-server-web\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Discard\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546373071
Jul 17 03:27:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:27.329Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"twa-tsgx09\", asset_id=\"107\", condition=\"Progressing\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"dashboard-metrics-scraper\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"unknown\"}" timestamp=1689546378744
Jul 17 03:27:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:27.385Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"plymouth-quit.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"service\"}" timestamp=1689546373907
Jul 17 03:27:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:27.422Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_init_container_info\", asset=\"twa-tsgx09\", asset_id=\"107\", container=\"firewall-init\", container_id=\"containerd://7fdf1ae3de55a32996a94700ce3b7e50c43ea5353cc51127020a7857881b08aa\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", image=\"registry.gdnt-cloud.website/tsg-init:v23.07-rc1-079e964\", image_id=\"sha256:c3a7da59ebe9d7390fc09b3b4d50c14674963d0d9616b631fe8029ed3bd0e7d1\", image_spec=\"registry.gdnt-cloud.website/tsg-init:v23.07-rc1-079e964\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", pod=\"tsg-traffic-engine-numa-0-firewall-d55dcdb59-dgxgd\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"48a4a0ca-42d3-4487-bc89-7615c4f62e66\"}" timestamp=1689546378744
Jul 17 03:27:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:27.468Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_commands_rejected_calls_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cmd=\"expire\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546383549
Jul 17 03:27:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:27.476Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_type\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.tsg-traffic-engine-numa-0.v5\", service=\"prometheus-kube-state-metrics\", type=\"helm.sh/release.v1\"}" timestamp=1689546378744
Jul 17 03:27:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:27.498Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"23\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546375810
Jul 17 03:27:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:27.535Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_metadata_generation\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"prometheus-alertmanager-6df7f457b6\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546378744
Jul 17 03:27:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:27.562Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth14ccc549\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546375810
Jul 17 03:27:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:27.564Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_working_set_bytes\", asset=\"twa-tsgx09\", asset_id=\"107\", container=\"kube-state-metrics\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546381665
Jul 17 03:27:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:27.612Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_labels\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-cm-policy-local-cache-wzw8p\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"57683893-10c5-486b-89a6-dbc1429ed965\"}" timestamp=1689546378744
Jul 17 03:27:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:27.658Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"42\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546375810
Jul 17 03:27:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:27.800Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_up_changes_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens6f0np0\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546382882
Jul 17 03:27:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:27.896Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_reads_total\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"metrics\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546381213
Jul 17 03:27:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:27.929Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_data_received_bytes_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_10\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546382882
Jul 17 03:27:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:27.948Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_system_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", container=\"prometheus-server\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546376200
Jul 17 03:27:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:27.949Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_writes_total\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"telegraf-security\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689546379302
Jul 17 03:27:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:27.971Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_packets_dropped_total\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"cadvisor\", interface=\"ens7f0np0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546381048
Jul 17 03:27:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:27.997Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_sockets\", asset=\"twa-tsgx09\", asset_id=\"107\", container=\"metrics\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546384071
Jul 17 03:27:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:28.044Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_labels\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.ipmi-exporter.v1\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546378744
Jul 17 03:27:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:28.049Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_labels\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"kubernetes-dashboard-csrf\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546375682
Jul 17 03:27:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:28.066Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"31\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546382882
Jul 17 03:27:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:28.082Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Udp_Offload_STM\", app_name=\"sapp4\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", vsysId=\"1\"}" timestamp=1689546375421
Jul 17 03:27:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:28.334Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"ext4\", name=\"data.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689546374114
Jul 17 03:27:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:28.634Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-ask-password-wall.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"service\"}" timestamp=1689546374114
Jul 17 03:27:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:28.669Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_rss\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"telegraf-security\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689546371865
Jul 17 03:27:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:28.687Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_errors_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", interface=\"net1\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546383792
Jul 17 03:27:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:28.690Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_created\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"local-path-provisioner\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546375682
Jul 17 03:27:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:28.700Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_running\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"prometheus-alertmanager-configmap-reload\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-alertmanager-6df7f457b6-pwr5q\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"25f2e5d4-c32e-4636-bf64-2b5006d78aad\"}" timestamp=1689546375682
Jul 17 03:27:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:28.730Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_net_dev_group\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens2f0np0\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546382882
Jul 17 03:27:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:28.777Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth43bf70e1\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546382882
Jul 17 03:27:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:28.786Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"telegraf\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Async\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689546375064
Jul 17 03:27:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:28.795Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_errs_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"vethd626638d\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546382882
Jul 17 03:27:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:28.801Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_info\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"metrics-server\", container_id=\"containerd://1f0a130785223ee0108e0cce80d86f231f30900e0a8cc24bfa398555fb8a97aa\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", image=\"docker.io/rancher/mirrored-metrics-server:v0.6.2\", image_id=\"sha256:25561daa666051dbc5204d6df0ed53882c24f906a2cb71357bf0b8771dc5ab11\", image_spec=\"rancher/mirrored-metrics-server:v0.6.2\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"metrics-server-54dc485875-mqqt7\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"b0c47859-1fba-4157-b772-f4e3a8229921\"}" timestamp=1689546375682
Jul 17 03:27:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:28.870Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"30\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546382882
Jul 17 03:27:30 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:30.983Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_sockets\", asset=\"msh-tsgx05\", asset_id=\"67\", container=\"local-path-provisioner\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546367590
Jul 17 03:27:30 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:30.996Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_status_start_time\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", job_name=\"helm-install-tsg-traffic-engine-numa-0\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546384068
Jul 17 03:27:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:31.111Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"msh-tsgx05\", asset_id=\"67\", container=\"ipmi-exporter\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"uninterruptible\"}" timestamp=1689546369251
Jul 17 03:27:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:31.112Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"nginx_vts_server_cache_total\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"Galaxy-hos-nginx-msh-dellr740-dt001\", endpoint_id=\"201\", host=\"192.168.20.251\", module=\"Galaxy-hos-nginx\", module_id=\"13\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", status=\"miss\"}" timestamp=1689546390873
Jul 17 03:27:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:31.187Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"42\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546380939
Jul 17 03:27:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:31.267Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"cm-policy-local-cache-redis-master-69967b5879-rj8lj\", project=\"TSG-OS\", project_id=\"7\", reason=\"NodeAffinity\", service=\"prometheus-kube-state-metrics\", uid=\"adf25328-be5b-4a8f-99a7-ca39872df08e\"}" timestamp=1689546384068
Jul 17 03:27:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:31.281Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"74\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546380939
Jul 17 03:27:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:31.314Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-virtual-net-vethd85974e3.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689546381015
Jul 17 03:27:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:31.395Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_files_free\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"shm\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/0b9ffe046fcf4cd096979ad8d16ea55f93485c7dc1eb6bc4c95cc6dcd20a6e79/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546380939
Jul 17 03:27:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:31.478Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_queue_length\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth39378f16\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546379093
Jul 17 03:27:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:31.492Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"rpc-statd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"service\"}" timestamp=1689546381015
Jul 17 03:27:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:31.514Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"11\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546380939
Jul 17 03:27:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:31.517Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"tsg_master_success_log_s\", app_name=\"tsg_master\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"traffic-engine\", line_name=\"sum\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", table_name=\"sendlog\", vsysId=\"1\"}" timestamp=1689546380031
Jul 17 03:27:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:31.517Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Tcp_Close_byK\", app_name=\"sapp4\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", vsysId=\"1\"}" timestamp=1689546380031
Jul 17 03:27:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:31.530Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"79\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546380939
Jul 17 03:27:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:31.563Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_threads\", asset=\"msh-tsgx05\", asset_id=\"67\", container=\"redis\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546375129
Jul 17 03:27:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:31.673Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sockets.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"target\"}" timestamp=1689546381015
Jul 17 03:27:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:31.729Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"110\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546380791
Jul 17 03:27:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:31.774Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"msh-tsgx05\", asset_id=\"67\", container=\"local-path-provisioner\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"uninterruptible\"}" timestamp=1689546367590
Jul 17 03:27:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:31.786Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db79\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546376711
Jul 17 03:27:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:31.792Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_spec_replicas\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"prometheus-kube-state-metrics\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546379276
Jul 17 03:27:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:31.824Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_scrape_collector_success\", asset=\"twa-tsgx03\", asset_id=\"95\", collector=\"powersupplyclass\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546379093
Jul 17 03:27:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:31.862Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_master_sync_in_progress\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", master_host=\"192.168.20.157\", master_port=\"7112\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546386438
Jul 17 03:27:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:31.877Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"iscsiuio.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"service\"}" timestamp=1689546386907
Jul 17 03:27:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:31.921Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"14\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546380791
Jul 17 03:27:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:31.936Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"3\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546380791
Jul 17 03:27:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:31.961Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_link_downed_total\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_3\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546379093
Jul 17 03:27:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:31.998Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_scrape_collector_success\", asset=\"twa-tsgx03\", asset_id=\"95\", collector=\"softnet\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546379093
Jul 17 03:27:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:32.011Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx05\", asset_id=\"67\", container=\"coredns\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Total\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546374189
Jul 17 03:27:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:32.071Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Udp_Bypass_Pkt\", app_name=\"sapp4\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", vsysId=\"1\"}" timestamp=1689546380718
Jul 17 03:27:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:32.088Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"19\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546380939
Jul 17 03:27:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:32.093Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"tmpfs\", name=\"run-k3s-containerd-io.containerd.grpc.v1.cri-sandboxes-389e5290b7a824756398cd2cdcc2529dbb214f0473c487f3de79a9346cbe4695-shm.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689546382644
Jul 17 03:27:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:32.100Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_memory_Active_bytes\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546379093
Jul 17 03:27:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:32.109Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"initrd-cleanup.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"oneshot\", type=\"service\"}" timestamp=1689546381015
Jul 17 03:27:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:32.113Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-e2593736\\\\x2de404\\\\x2d42cd\\\\x2da92c\\\\x2dc9652757c95a-volume\\\\x2dsubpaths-config\\\\x2dpath-dign\\\\x2dserver\\\\x2ddns-1.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689546382644
Jul 17 03:27:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:32.132Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-f590a5837d5ab6427be27f6d4fc128fb5b85feec0e43676a9400323a286ec0ed-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689546382644
Jul 17 03:27:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:32.152Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_reservation_limit_bytes\", asset=\"twa-tsgx03\", asset_id=\"95\", container=\"traefik\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546383123
Jul 17 03:27:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:32.197Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"34\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546379093
Jul 17 03:27:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:32.264Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_changes_total\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens8f0np0\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546379093
Jul 17 03:27:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:32.285Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_up_changes_total\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth662ec9f2\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546379093
Jul 17 03:27:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:32.285Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-841e23492d6c7cb72b6d23a21b11e15a6e93fcbc283f2b194cdc51818f13d2f2.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"scope\"}" timestamp=1689546386907
Jul 17 03:27:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:32.298Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:80-0000:80:01.1-0000:81:00.0-host0-target0:2:0-0:2:0:0-block-sda-sda1.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689546386907
Jul 17 03:27:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:32.326Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_start_time_seconds\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"dign-server-dns\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546382009
Jul 17 03:27:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:32.483Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"118\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546380605
Jul 17 03:27:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:32.673Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"87\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546379093
Jul 17 03:27:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:32.735Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_type\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.systemd-exporter.v1\", service=\"prometheus-kube-state-metrics\", type=\"helm.sh/release.v1\"}" timestamp=1689546379276
Jul 17 03:27:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:32.758Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"120\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546380791
Jul 17 03:27:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:32.776Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_metadata_generation\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"kubernetes-dashboard-66dd5b456\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546379276
Jul 17 03:27:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:32.800Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_user_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", container=\"firewall\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689546375959
Jul 17 03:27:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:32.806Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_dormant\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens8f1np1\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546380791
Jul 17 03:27:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:32.820Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_fifo_total\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens4f3np3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546380791
Jul 17 03:27:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:32.870Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"3\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546380791
Jul 17 03:27:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:32.921Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_state_started\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"helm\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-delete-tsg-traffic-engine-numa-1-cxlvk\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"1d75a15d-f414-4a89-93b6-193080fb66a4\"}" timestamp=1689546379276
Jul 17 03:27:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:33.003Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-tmpfiles-setup-dev.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689546382644
Jul 17 03:27:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:33.083Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_usage_seconds_total\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"dashboard-metrics-scraper\", cpu=\"total\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546378156
Jul 17 03:27:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:33.084Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_file_descriptors\", asset=\"twa-tsgx03\", asset_id=\"95\", container=\"dign-server-web\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546378053
Jul 17 03:27:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:33.114Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"rshim.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689546382644
Jul 17 03:27:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:33.130Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_info\", address=\"c2:9e:c4:e6:2a:a2\", asset=\"msh-tsgx11\", asset_id=\"111\", broadcast=\"ff:ff:ff:ff:ff:ff\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"virtio_dign_s\", duplex=\"full\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operstate=\"unknown\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546380791
Jul 17 03:27:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:33.169Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"47\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546380605
Jul 17 03:27:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:33.210Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-rfkill.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689546382644
Jul 17 03:27:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:33.271Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_file_descriptors\", asset=\"twa-tsgx03\", asset_id=\"95\", container=\"mrapm-device\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546375605
Jul 17 03:27:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:33.299Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"twa-tsgx03\", asset_id=\"95\", container=\"telegraf-security\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", state=\"uninterruptible\"}" timestamp=1689546373140
Jul 17 03:27:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:33.397Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"83\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546380605
Jul 17 03:27:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:33.512Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_usage_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", container=\"firewall\", cpu=\"total\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689546375959
Jul 17 03:27:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:33.525Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"twa-tsgx05\", asset_id=\"99\", container=\"dign-server-dns\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"cadvisor\", failure_type=\"pgfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"container\"}" timestamp=1689546372179
Jul 17 03:27:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:33.566Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-user-sessions.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689546382644
Jul 17 03:27:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:33.657Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_udp_queues\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", ip=\"v6\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", queue=\"rx\", service=\"prometheus-node-exporter\"}" timestamp=1689546380791
Jul 17 03:27:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:33.968Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"pcap-tsgx05\", asset_id=\"77\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx05\", endpoint_id=\"471\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"93\", nz_agent_id=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546384387
Jul 17 03:27:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:33.976Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"44\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546380605
Jul 17 03:27:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:34.233Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"metrics-server-54dc485875-pdn8r\", project=\"TSG-OS\", project_id=\"7\", reason=\"NodeAffinity\", service=\"prometheus-kube-state-metrics\", uid=\"3538cc3c-71c8-48b7-8345-977c5f681b51\"}" timestamp=1689546379894
Jul 17 03:27:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:34.267Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"111\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546380605
Jul 17 03:27:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:34.270Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"22\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546380605
Jul 17 03:27:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:34.288Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_drop_total\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethc8144301\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546380605
Jul 17 03:27:37 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:37.538Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"scrape_samples_scraped\", asset=\"msh-dellr740-olap004\", asset_id=\"17\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Hadoop-msh-dellr740-olap004-DataNode\", endpoint_id=\"273\", instance=\"192.168.20.224:9906\", job=\"273\", module=\"NC-Hadoop\", module_id=\"37\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546396625
Jul 17 03:27:37 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:37.992Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kafka_network_RequestMetrics_Errors_total\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-Kafka-twa-dellr740-dt001\", endpoint_id=\"105\", error=\"NONE\", module=\"DC-Kafka\", module_id=\"5\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", request=\"OffsetForLeaderEpoch\"}" timestamp=1689546397483
Jul 17 03:27:38 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:38.194Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kafka_server_socket_server_metrics_request_rate\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-Kafka-twa-dellr740-dt001\", endpoint_id=\"105\", listener=\"SASL_PLAINTEXT\", module=\"DC-Kafka\", module_id=\"5\", networkProcessor=\"2\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546397483
Jul 17 03:27:38 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:38.482Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kafka_network_RequestMetrics_Errors_total\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-Kafka-twa-dellr740-dt001\", endpoint_id=\"105\", error=\"NOT_COORDINATOR\", module=\"DC-Kafka\", module_id=\"5\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", request=\"DescribeGroups\"}" timestamp=1689546397483
Jul 17 03:27:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:40.493Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_created\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-mrzcpd-exporter-xgz49\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"301784b5-1927-45d7-84ae-687804ad7a5e\"}" timestamp=1689546400034
Jul 17 03:27:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:40.503Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled_time\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-tsg-traffic-engine-numa-0-qwtq5\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"8bcaaf2d-628e-4f10-872c-7a32ecf2946e\"}" timestamp=1689546400034
Jul 17 03:27:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:40.518Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_vl15_dropped_total\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_8\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546391647
Jul 17 03:27:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:40.746Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_init_container_status_restarts_total\", asset=\"twa-tsgx04\", asset_id=\"97\", container=\"tsg-dign-init\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"dign-server-7zllz\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"a76c8dce-9010-4080-b63c-3802527a42aa\"}" timestamp=1689546400034
Jul 17 03:27:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:40.788Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Tcp_Offload_Bit\", app_name=\"sapp4\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", vsysId=\"1\"}" timestamp=1689546393536
Jul 17 03:27:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:40.878Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_10_193_1_metric_scanTime_90th_percentile\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546399757
Jul 17 03:27:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:40.983Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_restarts_total\", asset=\"twa-tsgx04\", asset_id=\"97\", container=\"metrics\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"cm-policy-local-cache-redis-master-69967b5879-j7kh6\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"dc48faf7-72ec-4d62-807b-3dca8380f0aa\"}" timestamp=1689546400034
Jul 17 03:27:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:41.005Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_minStoreFileAge\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"proxy_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"c587d583e24dc588569dc8943be59489\", table=\"index_partfile_proxy_hos_bucket\", type=\"index\"}" timestamp=1689546399757
Jul 17 03:27:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:41.012Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_replicas\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"prometheus-server-6ff784d758\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546400034
Jul 17 03:27:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:41.047Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_info\", address=\"12:da:56:4c:09:2c\", asset=\"twa-tsgx04\", asset_id=\"97\", broadcast=\"ff:ff:ff:ff:ff:ff\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vetha33ccb73\", duplex=\"full\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operstate=\"up\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546391647
Jul 17 03:27:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:41.099Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"64\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546391647
Jul 17 03:27:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:41.177Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_lastMajorCompactionAge\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"e1aa3ed655d5e52c22fd2924c35179c0\", table=\"proxy_hos_bucket\", type=\"data\"}" timestamp=1689546399757
Jul 17 03:27:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:41.323Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_udp_queues\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", ip=\"v6\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", queue=\"rx\", service=\"prometheus-node-exporter\"}" timestamp=1689546391647
Jul 17 03:27:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:41.341Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"atd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"simple\", type=\"service\"}" timestamp=1689546395764
Jul 17 03:27:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:41.352Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_flags\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"eno36np3\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546391647
Jul 17 03:27:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:41.371Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_drop_total\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens2f1np1\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546391647
Jul 17 03:27:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:41.388Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-83d4b956d7e4183e848a5a98c1590f86f03c7a76fab7677319478f5186b5316a.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"scope\"}" timestamp=1689546395764
Jul 17 03:27:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:41.394Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_cpu_period\", asset=\"twa-tsgx04\", asset_id=\"97\", container=\"monit-stream\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546388797
Jul 17 03:27:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:41.410Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_10_194_1_metric_deleteBatchTime_min\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546399757
Jul 17 03:27:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:41.447Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_10_194_1_metric_tableReadQueryPerSecond_count\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546399757
Jul 17 03:27:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:41.484Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_MajorCompactionOutputSize_25th_percentile\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Server\"}" timestamp=1689546399757
Jul 17 03:27:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:41.610Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"initrd.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"target\"}" timestamp=1689546395764
Jul 17 03:27:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:41.640Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Bulkload_90th_percentile\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Server\"}" timestamp=1689546399757
Jul 17 03:27:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:42.057Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_replicaid\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"firewall_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"8137db44e9968b742fb004da91f36f2d\", table=\"index_time_firewall_hos_bucket\", type=\"index\"}" timestamp=1689546399757
Jul 17 03:27:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:42.128Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_CHILD_LINK_metric_getTime_mean\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546399757
Jul 17 03:27:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:42.445Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_load_average_10s\", asset=\"pcap-tsgx01\", asset_id=\"71\", container=\"firewall\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689546391224
Jul 17 03:27:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:42.540Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_errs_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens4f3np3\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546396242
Jul 17 03:27:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:42.560Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-e12645fd77e0267b7e4793e719ced23b1c706f0f70bf9a2b3489baaad542935d.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"scope\"}" timestamp=1689546400671
Jul 17 03:27:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:42.763Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"phy_rx_error_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", dev=\"virtio_dign_c\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"mrzcpd-exporter\"}" timestamp=1689546399613
Jul 17 03:27:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:42.781Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sysstat-collect.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"service\"}" timestamp=1689546400671
Jul 17 03:27:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:42.794Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_bytes_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"cadvisor\", interface=\"eno35np2\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546400525
Jul 17 03:27:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:42.803Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_tasks_current\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"plymouth-start.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689546400671
Jul 17 03:27:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:42.818Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_info\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", job_name=\"helm-install-tsg-traffic-engine-numa-1\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546395154
Jul 17 03:27:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:42.831Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_tasks_current\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-udev-settle.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689546400671
Jul 17 03:27:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:42.833Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"kube-multus-ds-t9bhj\", project=\"TSG-OS\", project_id=\"7\", reason=\"NodeLost\", service=\"prometheus-kube-state-metrics\", uid=\"095d5887-0ff1-468e-9463-a3765c458177\"}" timestamp=1689546395154
Jul 17 03:27:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:43.008Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"pcap-tsgx01\", asset_id=\"71\", condition=\"false\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-alertmanager-6df7f457b6-m8cl7\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"98daa3ea-45f0-41c2-be94-512fb3e2a016\"}" timestamp=1689546395154
Jul 17 03:27:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:43.038Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-udevd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"service\"}" timestamp=1689546400671
Jul 17 03:27:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:43.091Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_spec_paused\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"prometheus-pushgateway\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546395154
Jul 17 03:27:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:43.103Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", phase=\"Unknown\", pod=\"prometheus-pushgateway-bd476bcf9-scrs2\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"1d9c1047-30bc-4de8-bd00-07e7917cd6b7\"}" timestamp=1689546395154
Jul 17 03:27:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:43.122Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", cpu=\"110\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546396242
Jul 17 03:27:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:43.203Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"76\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546396242
Jul 17 03:27:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:43.205Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", container=\"dashboard-metrics-scraper\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Write\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546392782
Jul 17 03:27:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:43.218Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-da4d49d7df9a0fc7df687f77cd02d08194f47a4a9f39f030337bfd578385d473-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689546400671
Jul 17 03:27:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:43.270Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_dormant\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"lo\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546396242
Jul 17 03:27:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:43.293Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_colls_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens8f1np1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546396242
Jul 17 03:27:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:43.421Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2duuid-1d9a4674\\\\x2d4c98\\\\x2d4d1c\\\\x2d907a\\\\x2d2dada7a4ef1c.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689546400671
Jul 17 03:27:48 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:48.994Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_compactionsFailedCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", namespace=\"hbase\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"9e10db9aecfb71ee2a550bf485fa6d44\", table=\"namespace\", type=\"data\"}" timestamp=1689546407440
Jul 17 03:27:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:49.041Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_194_4_metric_appendTime_90th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546407440
Jul 17 03:27:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:49.059Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"java_lang_OperatingSystem_FreeSwapSpaceSize\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546407440
Jul 17 03:27:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:49.278Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_2_metric_flushMemstoreSize_25th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546407440
Jul 17 03:27:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:49.337Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_194_2_metric_scanTime_98th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546407440
Jul 17 03:27:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:49.428Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_maxFlushQueueSize\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"firewall_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"9c6a3a8e4919c9921fa19100cc22af9f\", table=\"index_partfile_firewall_hos_bucket\", type=\"index\"}" timestamp=1689546407440
Jul 17 03:27:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:49.566Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx10\", asset_id=\"109\", cpu=\"77\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546401797
Jul 17 03:27:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:49.703Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"java_lang_GarbageCollector_LastGcInfo_memoryUsageBeforeGc_used\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", key=\"Compressed Class Space\", module=\"DC-HBase\", module_id=\"3\", name=\"ConcurrentMarkSweep\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546407440
Jul 17 03:27:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:49.704Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_193_1_metric_flushOutputSize_num_ops\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546407440
Jul 17 03:27:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:49.799Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_info\", asset=\"twa-tsgx10\", asset_id=\"109\", cluster_ip=\"10.43.0.10\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"kube-dns\", uid=\"2f4ab68f-4664-4c86-b80c-fb5c042e5f9b\"}" timestamp=1689546399668
Jul 17 03:27:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:50.018Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_compactionsFailedCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", namespace=\"tsg_galaxy\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"ce0fe72f29202c7873dcf268f8d734d3\", table=\"gtpc_knowledge_base\", type=\"data\"}" timestamp=1689546407440
Jul 17 03:27:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:50.184Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"twa-tsgx10\", asset_id=\"109\", condition=\"Available\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"kubernetes-dashboard\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"false\"}" timestamp=1689546399668
Jul 17 03:27:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:50.299Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_physical_state_id\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_2\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546401797
Jul 17 03:27:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:50.378Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"initrd.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"target\"}" timestamp=1689546406554
Jul 17 03:27:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:50.444Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_193_1_metric_majorCompactionOutputSize_98th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546407440
Jul 17 03:27:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:50.467Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_flags\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens8f1np1\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546401797
Jul 17 03:27:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:50.492Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-ba106c44\\\\x2dd5c3\\\\x2d446a\\\\x2db10b\\\\x2d530b2492f73f-volume\\\\x2dsubpaths-telegraf-telegraf-0.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689546401057
Jul 17 03:27:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:50.501Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_state_id\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_1\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546401797
Jul 17 03:27:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:50.562Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_info\", address=\"a6:80:d9:0c:3c:c2\", asset=\"twa-tsgx10\", asset_id=\"109\", broadcast=\"ff:ff:ff:ff:ff:ff\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth23566533\", duplex=\"full\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operstate=\"up\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546401797
Jul 17 03:27:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:50.657Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_tsg_galaxy_table_gtpc_knowledge_base_metric_deleteBatchTime_median\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546407440
Jul 17 03:27:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:50.676Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_labels\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"tsg-traffic-engine-numa-1-firewall\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546399668
Jul 17 03:27:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:50.724Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_namespace_created\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546399668
Jul 17 03:27:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:50.842Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_scanCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"proxy_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"a3b7319ec286102d93f51022aa81c973\", table=\"index_partfile_proxy_hos_bucket\", type=\"index\"}" timestamp=1689546407440
Jul 17 03:27:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:50.857Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"tmpfs\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/var/lib/kubelet/pods/fd984c25-4c10-40eb-a03c-2c7c403cf652/volumes/kubernetes.io~projected/kube-api-access-2tf7t\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546401797
Jul 17 03:27:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:50.858Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_speed_bytes\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethbece5312\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546401797
Jul 17 03:27:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:50.928Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-ens3f1np1.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689546406554
Jul 17 03:27:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:50.950Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"shutdown.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"target\"}" timestamp=1689546406554
Jul 17 03:27:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:50.987Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_annotations\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"chart-values-mrzcpd-exporter\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546401568
Jul 17 03:27:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:51.037Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"90\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546406709
Jul 17 03:27:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:51.094Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"exporter-systemd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689546406554
Jul 17 03:27:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:51.096Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_maxFlushQueueSize\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"knowledge_base_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"56beb531d3f9e76ac417919ebcd4d1b6\", table=\"index_partfile_knowledge_base_hos_bucket\", type=\"index\"}" timestamp=1689546407440
Jul 17 03:27:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:51.126Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_compactionsCompletedCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"9d091b0c3b3083399f0cc4d765f30b7d\", table=\"session_record_hos_bucket\", type=\"data\"}" timestamp=1689546407440
Jul 17 03:27:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:51.186Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_annotations\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"systemd-exporter\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546401568
Jul 17 03:27:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:51.237Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_uname_info\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", domainname=\"(none)\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", machine=\"x86_64\", module=\"TSG-OS\", module_id=\"77\", nodename=\"twa-tsgx10\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", release=\"5.17.15-1.el8.x86_64\", service=\"prometheus-node-exporter\", sysname=\"Linux\", version=\"#1 SMP PREEMPT Wed Jun 15 02:07:24 UTC 2022\"}" timestamp=1689546401797
Jul 17 03:27:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:51.257Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"phy_tx_bits_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", dev=\"ens7f1np1\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"mrzcpd-exporter\"}" timestamp=1689546401366
Jul 17 03:27:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:51.270Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_timex_tick_seconds\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-olap003\", endpoint_id=\"235\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"yarn_resourcemanager,zookeeper,nacos,kafka,spark,druid_data,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546409713
Jul 17 03:27:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:51.390Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_dormant\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"virtio_dign_c\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546406709
Jul 17 03:27:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:51.527Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_reads_bytes_total\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"prometheus-alertmanager\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546389024
Jul 17 03:27:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:51.553Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"56\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546406709
Jul 17 03:27:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:51.607Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"process_open_fds\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546407556
Jul 17 03:27:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:51.623Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2dpath-pci\\\\x2d0000:81:00.0\\\\x2dscsi\\\\x2d0:2:0:0\\\\x2dpart1.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689546406554
Jul 17 03:27:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:51.644Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:40-0000:40:01.1-0000:41:00.1-net-ens3f1np1.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689546406554
Jul 17 03:27:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:51.782Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_cache\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"exporter\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546397978
Jul 17 03:27:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:51.811Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"twa-tsgx08\", asset_id=\"105\", condition=\"false\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-tsg-traffic-engine-numa-0-zd575\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"89cff922-a619-4acb-bbbe-b8e262b508b2\"}" timestamp=1689546401568
Jul 17 03:27:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:51.838Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_frequency_min_hertz\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"4\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546406709
Jul 17 03:27:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:51.866Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_packets_received_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546406709
Jul 17 03:27:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:51.900Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"kube-multus\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", failure_type=\"pgfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"container\"}" timestamp=1689546387934
Jul 17 03:27:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:51.928Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_processes\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"traefik\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546394567
Jul 17 03:27:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:51.984Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx10\", asset_id=\"109\", cpu=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546401797
Jul 17 03:27:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:51.995Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db52\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546407556
Jul 17 03:27:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:52.016Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"scrape_series_added\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_instance=\"10.42.0.123:9273\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", vsysId=\"1\"}" timestamp=1689546408675
Jul 17 03:27:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:52.035Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-fe1b9683\\\\x2d356b\\\\x2d448d\\\\x2d8311\\\\x2d9cb2e5a90800-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2drskgg.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689546406554
Jul 17 03:27:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:52.477Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_commands_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", cmd=\"zrangebyscore\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546399367
Jul 17 03:27:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:52.553Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_usage_bytes\", asset=\"pcap-tsgx11\", asset_id=\"115\", container=\"telegraf\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689546406177
Jul 17 03:27:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:52.600Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", container=\"prometheus-pushgateway\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"running\"}" timestamp=1689546409558
Jul 17 03:27:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:52.602Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Unknown\", pod=\"coredns-6658b4d4d7-tm6df\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"9eafc211-7f7b-4e70-b713-ce422850dd16\"}" timestamp=1689546404478
Jul 17 03:27:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:52.618Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_observed_generation\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-1-firewall-656745dd47\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546404478
Jul 17 03:27:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:52.618Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db248\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546399367
Jul 17 03:27:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:52.653Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth94114e67\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546409337
Jul 17 03:27:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:52.666Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db209\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546399367
Jul 17 03:27:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:52.963Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_last_seen\", asset=\"pcap-tsgx11\", asset_id=\"115\", container=\"telegraf-security\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689546410088
Jul 17 03:27:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:52.977Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_info\", asset=\"pcap-tsgx11\", asset_id=\"115\", configmap=\"kube-root-ca.crt\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"consul\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546404478
Jul 17 03:27:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:53.036Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_created\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_endpoint=\"prometheus-node-exporter\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546404478
Jul 17 03:27:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:53.074Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link_mode\", asset=\"pcap-dellr740-dt002\", asset_id=\"33\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"vethbb70265\", endpoint=\"NC-Gohangout-pcap-dellr740-dt002\", endpoint_id=\"173\", module=\"DC-Gohangout\", module_id=\"35\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546412666
Jul 17 03:27:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:53.074Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", cpu=\"23\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546409337
Jul 17 03:27:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:53.090Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_packets_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens6f0np0\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546409337
Jul 17 03:27:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:53.091Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_id\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth736616a4\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546409337
Jul 17 03:27:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:53.113Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db158\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546399367
Jul 17 03:27:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:53.116Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_init_container_status_restarts_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", container=\"install-multus-binary\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"kube-multus-ds-rvzdl\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"ac103daf-2fd5-4026-ae82-9b9eca69f2cd\"}" timestamp=1689546404478
Jul 17 03:27:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:53.132Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"pcap-tsgx11\", asset_id=\"115\", condition=\"unknown\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"metrics-server-54dc485875-pqvd7\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"aed8aa53-e603-467c-8ed7-d385c606efb7\"}" timestamp=1689546404478
Jul 17 03:27:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:53.132Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_id\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth561c9aba\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546409337
Jul 17 03:27:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:53.167Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"tsg_master_del\", app_name=\"tsg_master\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"traffic-engine\", line_name=\"app_id_dict\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", table_name=\"sync_exdata\", vsysId=\"1\"}" timestamp=1689546404735
Jul 17 03:27:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:53.179Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_queue_length\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"virtio_dign_c\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546409337
Jul 17 03:27:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:53.194Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_timex_maxerror_seconds\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546409337
Jul 17 03:27:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:53.243Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_annotations\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.mrzcpd-exporter.v1\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546408950
Jul 17 03:27:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:53.286Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_ips\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", ip=\"10.42.0.46\", ip_family=\"4\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"cm-policy-local-cache-redis-master-69967b5879-mfs5z\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"9b559a6c-7e7e-492f-aeb3-f47a8be58f12\"}" timestamp=1689546404478
Jul 17 03:27:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:53.304Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", cpu=\"6\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546409337
Jul 17 03:27:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:53.464Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_bytes_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"cadvisor\", interface=\"eno34np1\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546407719
Jul 17 03:27:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:53.502Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", cpu=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546409337
Jul 17 03:27:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:53.556Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_max_usage_bytes\", asset=\"msh-tsgx12\", asset_id=\"113\", container=\"exporter-merger\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546391459
Jul 17 03:27:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:53.558Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-rdma-devices-mlx5_3:1-umad.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689546405496
Jul 17 03:27:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:53.618Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_oom_events_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", container=\"prometheus-alertmanager\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546409606
Jul 17 03:27:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:53.655Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_info\", asset=\"pcap-tsgx11\", asset_id=\"115\", configmap=\"cm-cache-extra-scripts\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546404478
Jul 17 03:27:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:53.677Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_complete\", asset=\"msh-tsgx08\", asset_id=\"65\", condition=\"unknown\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", job_name=\"helm-install-tsg-traffic-engine-numa-1\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546408950
Jul 17 03:27:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:53.683Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_netstat_Tcp_RetransSegs\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546409337
Jul 17 03:27:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:53.747Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", cpu=\"26\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546409337
Jul 17 03:27:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:53.861Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_metadata_generation\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"cm-policy-local-cache-redis-master\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546402058
Jul 17 03:27:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:53.928Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"2\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546408328
Jul 17 03:27:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:54.292Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"msh-tsgx12\", asset_id=\"113\", cpu=\"43\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546403391
Jul 17 03:27:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:54.396Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_discards_transmitted_total\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_8\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546408328
Jul 17 03:27:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:54.469Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-update-done.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"oneshot\", type=\"service\"}" timestamp=1689546411179
Jul 17 03:27:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:54.615Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx08\", asset_id=\"65\", container=\"dashboard-metrics-scraper\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Total\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546384468
Jul 17 03:27:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:54.635Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"50\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546408328
Jul 17 03:27:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:54.649Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx08\", asset_id=\"65\", cpu=\"102\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546408328
Jul 17 03:27:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:54.678Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_persistentvolume_status_phase\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", persistentvolume=\"pvc-625cdf6c-e629-442c-9ba2-7d1e8c98f868\", phase=\"Failed\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546408950
Jul 17 03:27:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:54.700Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cpu=\"1\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546399409
Jul 17 03:27:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:54.730Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled_time\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-delete-tsg-traffic-engine-numa-1-pv6hh\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"ee9f4174-ab05-411d-a216-578ec25bab94\"}" timestamp=1689546408950
Jul 17 03:27:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:54.735Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx08\", asset_id=\"65\", container=\"redis\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Total\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546390212
Jul 17 03:27:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:54.746Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_info\", asset=\"msh-tsgx08\", asset_id=\"65\", created_by_kind=\"DaemonSet\", created_by_name=\"kube-multus-ds\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", host_ip=\"10.10.20.168\", host_network=\"true\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", node=\"localhost\", nz_agent_id=\"3\", pod=\"kube-multus-ds-t7twb\", pod_ip=\"10.10.20.168\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"4d27b507-e334-47c3-b9ec-ca1340a8a907\"}" timestamp=1689546408950
Jul 17 03:27:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:54.749Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_packets_received_total\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_5\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546408328
Jul 17 03:27:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:54.768Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_multicast_total\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens4f2np2\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546408328
Jul 17 03:27:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:54.771Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Udp_Bit\", app_name=\"sapp4\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", vsysId=\"1\"}" timestamp=1689546407622
Jul 17 03:27:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:54.774Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_metadata_generation\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"prometheus-alertmanager\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546402058
Jul 17 03:27:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:54.781Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx08\", asset_id=\"65\", cpu=\"71\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546408328
Jul 17 03:27:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:54.799Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_oom_events_total\", asset=\"msh-tsgx08\", asset_id=\"65\", container=\"metrics-server\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546386305
Jul 17 03:27:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:54.801Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_node_status_condition\", asset=\"pcap-tsgx07\", asset_id=\"85\", condition=\"PIDPressure\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", node=\"localhost\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"true\"}" timestamp=1689546402058
Jul 17 03:27:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:54.842Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_limit_bytes\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689546410608
Jul 17 03:27:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:54.854Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_speed_bytes\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"eno34np1\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546408328
Jul 17 03:27:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:54.899Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cpu=\"1\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546399409
Jul 17 03:27:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:54.910Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cpu=\"49\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546399409
Jul 17 03:27:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:54.928Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_sockstat_TCP_inuse\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546408328
Jul 17 03:27:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:54.946Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_address_assign_type\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"vethf1e4e7f4\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546399409
Jul 17 03:27:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:54.947Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth55feb6e8\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546399409
Jul 17 03:27:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:54.961Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db216\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546398659
Jul 17 03:27:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:54.967Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_fifo_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens3f0np0\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546399409
Jul 17 03:27:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:55.002Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"ext4\", name=\"var-lib-kubelet-pods-37750f9f\\\\x2d766e\\\\x2d4714\\\\x2da437\\\\x2dd6b2d6f2c90a-volume\\\\x2dsubpaths-config\\\\x2dvolume-telegraf\\\\x2dsecurity-1.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689546400884
Jul 17 03:27:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:55.005Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_daemonset_annotations\", asset=\"msh-tsgx08\", asset_id=\"65\", daemonset=\"kube-multus-ds\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546408950
Jul 17 03:27:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:55.015Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_queue_length\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens7f0np0\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546399409
Jul 17 03:27:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:55.025Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx08\", asset_id=\"65\", cpu=\"8\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546408328
Jul 17 03:27:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:55.041Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_last_terminated_reason\", asset=\"msh-tsgx12\", asset_id=\"113\", container=\"metrics-server\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"metrics-server-54dc485875-c275t\", project=\"TSG-OS\", project_id=\"7\", reason=\"Unknown\", service=\"prometheus-kube-state-metrics\", uid=\"b2135498-df72-42aa-b8d4-4ad10a2b2285\"}" timestamp=1689546408663
Jul 17 03:27:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:55.058Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_created\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"local-path-provisioner-957fdf8bc-8p6hf\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"a2905d97-aef8-4b28-aa36-33d2594480ee\"}" timestamp=1689546408663
Jul 17 03:27:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:55.073Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_unicast_packets_transmitted_total\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_2\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546403391
Jul 17 03:27:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:55.090Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"nfs-client.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"target\"}" timestamp=1689546411179
Jul 17 03:27:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:55.291Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"62\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546399409
Jul 17 03:27:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:55.319Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cpu=\"44\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546399409
Jul 17 03:27:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:55.333Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_packets_transmitted_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"mlx5_4\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546399409
Jul 17 03:27:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:55.471Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_cpu_seconds_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-pod17060853_6cdd_48b9_836a_8dba7f276cf9.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"slice\"}" timestamp=1689546400884
Jul 17 03:27:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:56.324Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_annotations\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"shaping-announce-port-tsg-traffic-engine-numa-1\", uid=\"ed40b3f0-67a2-459b-afef-cc4c8849dd6a\"}" timestamp=1689546413389
Jul 17 03:27:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:56.383Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"tsg_master_sce\", app_name=\"tsg_master\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"traffic-engine\", line_name=\"activing_fail\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", table_name=\"ctrl_service\", vsysId=\"1\"}" timestamp=1689546410331
Jul 17 03:27:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:56.399Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", pod=\"tsg-traffic-engine-numa-0-firewall-5bf9999fcf-5s2lq\", project=\"TSG-OS\", project_id=\"7\", reason=\"Shutdown\", service=\"prometheus-kube-state-metrics\", uid=\"aebe8d09-6d5d-4eda-b90d-7be24e2da709\"}" timestamp=1689546413389
Jul 17 03:27:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:56.692Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_complete\", asset=\"pcap-tsgx03\", asset_id=\"73\", condition=\"unknown\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", job_name=\"helm-install-traefik\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546413389
Jul 17 03:27:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:56.722Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_annotations\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-server-6ff784d758-2bs79\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"d1430253-8c7e-449d-9f2a-dd552c402bae\"}" timestamp=1689546413389
Jul 17 03:27:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:56.753Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-netns-cni\\\\x2dba614b88\\\\x2d53fd\\\\x2d8f16\\\\x2d7d70\\\\x2d6b2fd8d613f4.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689546403372
Jul 17 03:27:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:56.791Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_compressed_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens6f1np1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546405883
Jul 17 03:27:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:56.837Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"12\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546405883
Jul 17 03:27:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:56.933Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"vmtoolsd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689546403372
Jul 17 03:27:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:57.091Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_max_fds\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"watchdog.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689546403372
Jul 17 03:27:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:57.158Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"clickhouse_os_nice_time\", asset=\"msh-dellr740-olap001\", asset_id=\"11\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Clickhouse-msh-dellr740-olap001\", endpoint_id=\"71\", module=\"NC-Clickhouse\", module_id=\"29\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546415798
Jul 17 03:27:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:57.180Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kdump.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689546403372
Jul 17 03:27:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:57.298Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Tcp_Link_Double\", app_name=\"sapp4\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", vsysId=\"1\"}" timestamp=1689546410331
Jul 17 03:27:58 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:58.063Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", cpu=\"32\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"OLAP-Node-Exporter-pcap-dellr740-dt001\", endpoint_id=\"251\", mode=\"softirq\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546417560
Jul 17 03:27:58 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:58.544Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_scaling_frequency_max_hertz\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", cpu=\"35\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"OLAP-Node-Exporter-pcap-dellr740-dt001\", endpoint_id=\"251\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546417560
Jul 17 03:27:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:59.681Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_compressed_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth358d74e7\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546409083
Jul 17 03:27:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:59.687Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_rejected_connections_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546405071
Jul 17 03:27:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:59.696Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_errs_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"eno2\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546409083
Jul 17 03:27:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:59.701Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-286e9e2e78631e03427a533952967197d6ba74291280491099bf1d3700b1200c-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689546404879
Jul 17 03:27:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:59.779Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_start_time_seconds\", asset=\"msh-tsgx01\", asset_id=\"51\", container=\"monit-stream\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546414593
Jul 17 03:27:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:59.792Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_last_terminated_reason\", asset=\"msh-tsgx01\", asset_id=\"51\", container=\"prometheus-node-exporter\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-node-exporter-t6jf2\", project=\"TSG-OS\", project_id=\"7\", reason=\"Unknown\", service=\"prometheus-kube-state-metrics\", uid=\"21204ec2-df19-472e-9ae1-06104ab28d13\"}" timestamp=1689546407425
Jul 17 03:27:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:59.837Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_bytes_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"cadvisor\", interface=\"eno34np1\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546414231
Jul 17 03:27:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:59.935Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_multicast_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens4f2np2\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546409083
Jul 17 03:27:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:27:59.965Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_commands_failed_calls_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cmd=\"select\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546405071
Jul 17 03:28:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:00.018Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_drop_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth9b710207\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546409083
Jul 17 03:28:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:00.097Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2did-scsi\\\\x2dSDELL_PERC_H345_Adp_009dff86e3be672b2c00ecd485f0a7ce\\\\x2dpart2.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689546404879
Jul 17 03:28:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:00.116Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_packets_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"cadvisor\", interface=\"ens4f2np2\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546414231
Jul 17 03:28:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:00.341Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"46\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546409083
Jul 17 03:28:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:00.368Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-sda1.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689546404879
Jul 17 03:28:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:01.287Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_metadata_resource_version\", asset=\"msh-tsgx01\", asset_id=\"51\", configmap=\"cluster-dns\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546407425
Jul 17 03:28:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:01.341Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_receive_remote_physical_errors_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_4\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546409083
Jul 17 03:28:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:01.385Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_frame_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens6f1np1\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546409083
Jul 17 03:28:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:01.435Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_usage_bytes\", asset=\"msh-tsgx01\", asset_id=\"51\", container=\"mrapm-device\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546400462
Jul 17 03:28:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:02.452Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"openibd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"service\"}" timestamp=1689546408447
Jul 17 03:28:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:02.539Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2did-scsi\\\\x2d362cea7f084826d002c2b6d2f9e5d290c.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689546408447
Jul 17 03:28:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:02.548Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"msh-tsgx02\", asset_id=\"63\", condition=\"false\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-traefik-ph7rq\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"eacc1d44-839a-43ee-bf5c-0bccf9558d70\"}" timestamp=1689546411412
Jul 17 03:28:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:02.564Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"msh-tsgx02\", asset_id=\"63\", condition=\"unknown\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"ipmi-exporter-6f8464dd94-p7bhx\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"95d4eb15-f514-4daa-ac28-6317addaafb6\"}" timestamp=1689546411412
Jul 17 03:28:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:03.071Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled_time\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-systemd-exporter-gn8qh\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"3ac33a80-602d-4e22-a3ab-07460cbdb515\"}" timestamp=1689546411412
Jul 17 03:28:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:04.058Z caller=dedupe.go:112 component=remote level=info remote_name=52db6f url=http://10.10.10.159:10090/nz-agent/prometheus/proxy/api/v1/write msg="Dropped sample for series that was not explicitly dropped via relabelling" ref=4436650
Jul 17 03:28:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:04.082Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_bytes_total\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", interface=\"ens4f3np3\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546409854
Jul 17 03:28:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:04.473Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"net_conntrack_dialer_conn_closed_total\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", dialer_name=\"asset_ping_85\", instance=\"10.10.20.159:10090\", type=\"global\"}" timestamp=1689546422990
Jul 17 03:28:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:04.500Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"prometheus_target_scrape_pool_target_limit\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", instance=\"10.10.20.159:10090\", scrape_job=\"149\", type=\"global\"}" timestamp=1689546422990
Jul 17 03:28:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:04.734Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"net_conntrack_dialer_conn_failed_total\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", dialer_name=\"351\", instance=\"10.10.20.159:10090\", reason=\"timeout\", type=\"global\"}" timestamp=1689546422990
Jul 17 03:28:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:06.713Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_colls_total\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"eno34np1\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546423910
Jul 17 03:28:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:06.730Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cpu=\"30\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546423910
Jul 17 03:28:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:07.139Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cpu=\"8\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546423910
Jul 17 03:28:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:07.187Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_scaling_frequency_max_hertz\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", cpu=\"28\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Spark-msh-dellr740-olap003\", endpoint_id=\"13\", module=\"NC-Spark\", module_id=\"47\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546426092
Jul 17 03:28:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:07.215Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_compressed_total\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth444895a4\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546423910
Jul 17 03:28:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:07.566Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_down_changes_total\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethac72a555\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546423910
Jul 17 03:28:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:07.782Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_scrape_collector_duration_seconds\", asset=\"twa-tsgx01\", asset_id=\"91\", collector=\"sockstat\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546423910
Jul 17 03:28:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:07.985Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_other_sys_bytes\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Spark-msh-dellr740-olap003\", endpoint_id=\"13\", module=\"NC-Spark\", module_id=\"47\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546426092
Jul 17 03:28:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:08.166Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cpu=\"118\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546423910
Jul 17 03:28:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:08.210Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_drop_total\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens8f1np1\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546423910
Jul 17 03:28:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:08.226Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cpu=\"56\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546423910
Jul 17 03:28:09 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:09.600Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_maxStoreFileAge\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"knowledge_base_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"3100933be035cdb4ed680b9fd0ce6e2f\", table=\"index_filename_knowledge_base_hos_bucket\", type=\"index\"}" timestamp=1689546428805
Jul 17 03:28:09 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:09.705Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_getCount\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"session_record_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"ec653ac551c553eeba75d8187e9fc607\", table=\"index_filename_session_record_hos_bucket\", type=\"index\"}" timestamp=1689546428805
Jul 17 03:28:09 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:09.748Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_maxCompactionQueueSize\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"assessment_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"3c356a13043f7e253e70b459f09ed06f\", table=\"index_filename_assessment_hos_bucket\", type=\"index\"}" timestamp=1689546428805
Jul 17 03:28:09 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:09.848Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_minStoreFileAge\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", module=\"DC-HBase\", module_id=\"3\", namespace=\"SYSTEM\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"99211ccab1648d1974131d70ea51a2d7\", table=\"LOG\", type=\"data\"}" timestamp=1689546428805
Jul 17 03:28:09 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:09.910Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_flushesQueuedCount\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"proxy_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"837d0bb3c8b8aa4665a4ace2b6e2c6ed\", table=\"index_filename_proxy_hos_bucket\", type=\"index\"}" timestamp=1689546428805
Jul 17 03:28:09 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:09.956Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_memStoreSize\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"620634bf34681e6b8c305b20cced5b79\", table=\"troubleshooting_hos_bucket\", type=\"data\"}" timestamp=1689546428805
Jul 17 03:28:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:10.593Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_compactionsQueuedCount\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"default\", project=\"OLAP\", project_id=\"3\", region=\"c3ec92d9078ffa0a16297912d37136cc\", table=\"index_filename_default\", type=\"index\"}" timestamp=1689546428805
Jul 17 03:28:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:10.741Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_scanCount\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"session_record_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"3e6dab555ce48eac0eae0a9fa8102438\", table=\"index_filename_session_record_hos_bucket\", type=\"index\"}" timestamp=1689546428805
Jul 17 03:28:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:10.887Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_getCount\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"firewall_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"700a07cb70ff9f2c61808c3e4df6599d\", table=\"index_filename_firewall_hos_bucket\", type=\"index\"}" timestamp=1689546428805
Jul 17 03:28:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:11.291Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_1_metric_putBatchTime_99th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546430150
Jul 17 03:28:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:11.313Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"java_lang_OperatingSystem_ProcessCpuLoad\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546430150
Jul 17 03:28:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:11.384Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_filteredReadRequestCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"session_record_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"64ac4b5864b8b0132c1e942230dfc553\", table=\"index_time_session_record_hos_bucket\", type=\"index\"}" timestamp=1689546430150
Jul 17 03:28:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:11.402Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_193_1_metric_majorCompactionOutputSize_median\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546430150
Jul 17 03:28:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:11.479Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_2_metric_putBatchTime_99th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546430150
Jul 17 03:28:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:11.496Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_1_metric_storeFileCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546430150
Jul 17 03:28:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:11.508Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_194_3_metric_splitTime_mean\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546430150
Jul 17 03:28:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:11.835Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_1_metric_appendTime_num_ops\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546430150
Jul 17 03:28:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:11.978Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_3_metric_deleteTime_90th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546430150
Jul 17 03:28:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:12.161Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-dellr740-dt002\", asset_id=\"39\", cpu=\"31\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"OLAP-Node-Exporter-twa-dellr740-dt002\", endpoint_id=\"259\", mode=\"system\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546432055
Jul 17 03:28:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:12.283Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_194_1_metric_deleteBatchTime_95th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546430150
Jul 17 03:28:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:12.485Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_numBytesCompactedCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"proxy_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"a6078361d4a0ffaafa787562c158d6fc\", table=\"index_time_proxy_hos_bucket\", type=\"index\"}" timestamp=1689546430150
Jul 17 03:28:13 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:13.088Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_1_metric_flushTime_98th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546430150
Jul 17 03:28:13 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:13.139Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_4_metric_compactionInputSize_99th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546430150
Jul 17 03:28:13 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:13.148Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_193_1_metric_scanTime_25th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546430150
Jul 17 03:28:13 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:13.161Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_193_1_metric_majorCompactionInputFileCount_98th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546430150
Jul 17 03:28:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:15.261Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_ulimits_soft\", asset=\"twa-tsgx07\", asset_id=\"103\", container=\"coredns\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", ulimit=\"max_open_files\"}" timestamp=1689546418614
Jul 17 03:28:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:15.403Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"111\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546421230
Jul 17 03:28:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:15.438Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"tmpfs\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/data/var/lib/kubelet/pods/d909c7f6-8616-4c21-913b-225bb207a2ea/volumes/kubernetes.io~projected/kube-api-access-4kbbw\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546421230
Jul 17 03:28:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:15.550Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_free_bytes\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", fstype=\"ext4\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/data\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546421230
Jul 17 03:28:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:15.619Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_info\", address=\"3a:fe:c0:52:a0:da\", asset=\"twa-tsgx07\", asset_id=\"103\", broadcast=\"ff:ff:ff:ff:ff:ff\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"cnibr0\", duplex=\"unknown\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operstate=\"up\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546421230
Jul 17 03:28:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:16.396Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_queue_length\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth9bf7720f\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546421230
Jul 17 03:28:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:16.435Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_ports\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_endpoint=\"prometheus-server\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", port_name=\"http\", port_number=\"9090\", port_protocol=\"TCP\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546428076
Jul 17 03:28:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:16.462Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"twa-tsgx07\", asset_id=\"103\", container=\"prometheus-alertmanager-configmap-reload\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"uninterruptible\"}" timestamp=1689546420695
Jul 17 03:28:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:16.514Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_persistentvolume_annotations\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", persistentvolume=\"pvc-6597e086-e4ec-4d42-a234-02d6678a8f83\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546428076
Jul 17 03:28:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:16.667Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_complete\", asset=\"twa-tsgx07\", asset_id=\"103\", condition=\"unknown\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", job_name=\"helm-install-systemd-exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546428076
Jul 17 03:28:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:16.744Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_virtual_memory_bytes\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"atd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689546427046
Jul 17 03:28:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:16.873Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Udp_Close_byK\", app_name=\"sapp4\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", vsysId=\"1\"}" timestamp=1689546431690
Jul 17 03:28:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:16.889Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Udp_Offload_STM\", app_name=\"sapp4\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", vsysId=\"1\"}" timestamp=1689546420513
Jul 17 03:28:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:16.982Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_working_set_bytes\", asset=\"twa-tsgx07\", asset_id=\"103\", container=\"kube-multus\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546423171
Jul 17 03:28:17 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:17.003Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"twa-tsgx07\", asset_id=\"103\", container=\"kube-multus\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"stopped\"}" timestamp=1689546423171
Jul 17 03:28:17 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:17.046Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_cpu_seconds_total\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-podc435f080_d1a5_46a3_868f_b5d8209ce2ea.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"slice\"}" timestamp=1689546427046
Jul 17 03:28:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:19.796Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_netstat_Udp_InDatagrams\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546437612
Jul 17 03:28:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:19.877Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_usage_bytes\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689546431577
Jul 17 03:28:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:19.915Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-d45e136aa511981e1cddf3b122cea374a162c45d8a566542d022c884d7d97985-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689546424356
Jul 17 03:28:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:19.943Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-49c57b7aadcedcc04531126f7d534511218174bb124e5ac3cd382d5f71404099-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689546424356
Jul 17 03:28:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:20.497Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", container=\"dign-server-dns\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"stopped\"}" timestamp=1689546433420
Jul 17 03:28:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:20.555Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_ports\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_endpoint=\"prometheus-server\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", port_name=\"http\", port_number=\"9090\", port_protocol=\"TCP\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546433828
Jul 17 03:28:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:20.577Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_flags\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens6f0np0\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546428263
Jul 17 03:28:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:20.579Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_cache\", asset=\"twa-tsgx06\", asset_id=\"101\", container=\"prometheus-server-configmap-reload\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546427178
Jul 17 03:28:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:20.592Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"plymouth-quit.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689546424356
Jul 17 03:28:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:20.613Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_id\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens3f0np0\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546428263
Jul 17 03:28:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:20.662Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", container=\"configmap-reloader\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Discard\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546433983
Jul 17 03:28:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:20.673Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"mrzcpd_hwdb_setup.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"service\"}" timestamp=1689546430657
Jul 17 03:28:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:20.732Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_usage_bytes\", asset=\"twa-tsgx06\", asset_id=\"101\", container=\"exporter\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546418215
Jul 17 03:28:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:21.068Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-b9ed569c7da51837144927a3eea70259ac129aa89ffd5e8e96518bf89314ca06.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"scope\"}" timestamp=1689546424356
Jul 17 03:28:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:21.120Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"21\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546437612
Jul 17 03:28:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:21.140Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"77\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546437612
Jul 17 03:28:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:21.141Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", container=\"coredns\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Discard\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546432991
Jul 17 03:28:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:21.167Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"106\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546428263
Jul 17 03:28:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:21.187Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_observed_generation\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"systemd-exporter\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546433828
Jul 17 03:28:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:21.237Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"87\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546428263
Jul 17 03:28:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:21.256Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_transmit_wait_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"mlx5_7\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546428263
Jul 17 03:28:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:21.263Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_files\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"shm\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/89e2d284072c918459494a673de1758c0e35cfa8d573f12839747dd5eb98d0a1/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546428263
Jul 17 03:28:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:21.273Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"tmpfs\", name=\"var-lib-kubelet-pods-7fc8eb18\\\\x2ddf9b\\\\x2d4e31\\\\x2daa08\\\\x2de338eee64b81-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2dzgr9m.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689546424356
Jul 17 03:28:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:21.507Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"92\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546428263
Jul 17 03:28:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:21.648Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"gssproxy.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"forking\", type=\"service\"}" timestamp=1689546430657
Jul 17 03:28:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:21.668Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"pcap-tsgx10\", asset_id=\"89\", condition=\"Progressing\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"tsg-traffic-engine-numa-0-firewall\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"unknown\"}" timestamp=1689546433828
Jul 17 03:28:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:21.757Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_namespace_status_phase\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", phase=\"Terminating\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546433828
Jul 17 03:28:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:22.017Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", cpu=\"19\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"OLAP-Node-Exporter-twa-dellr740-dt001\", endpoint_id=\"257\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546441406
Jul 17 03:28:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:22.164Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", cpu=\"25\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"OLAP-Node-Exporter-twa-dellr740-dt001\", endpoint_id=\"257\", mode=\"iowait\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546441406
Jul 17 03:28:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:22.236Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda2\", endpoint=\"OLAP-Node-Exporter-twa-dellr740-dt001\", endpoint_id=\"257\", fstype=\"xfs\", module=\"OLAP-Node-Exporter\", module_id=\"51\", mountpoint=\"/boot\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546441406
Jul 17 03:28:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:23.708Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"k3s.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"notify\", type=\"service\"}" timestamp=1689546438416
Jul 17 03:28:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:23.733Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", pod=\"tsg-traffic-engine-numa-0-firewall-6cb6b98c7c-msh7v\", project=\"TSG-OS\", project_id=\"7\", reason=\"NodeLost\", service=\"prometheus-kube-state-metrics\", uid=\"405e3b50-5fa3-4f39-9405-ac337f946135\"}" timestamp=1689546440068
Jul 17 03:28:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:23.753Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_drop_total\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"vethb06b3a71\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546434790
Jul 17 03:28:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:23.775Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"msh-tsgx09\", asset_id=\"55\", cpu=\"115\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546434790
Jul 17 03:28:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:23.794Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dracut-pre-mount.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"service\"}" timestamp=1689546438416
Jul 17 03:28:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:24.218Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_packets_transmitted_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"mlx5_11\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546440160
Jul 17 03:28:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:24.238Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_changes_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens2f1np1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546440160
Jul 17 03:28:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:24.313Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_annotations\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-ipmi-exporter-wk46t\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"7a3c1563-fe1c-4a2b-b7d1-5ba87553013e\"}" timestamp=1689546431122
Jul 17 03:28:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:24.401Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_queue_length\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens4f0np0\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546440160
Jul 17 03:28:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:24.419Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_max_fds\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"rdma-ndd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689546431102
Jul 17 03:28:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:24.459Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_system_seconds_total\", asset=\"msh-tsgx09\", asset_id=\"55\", container=\"kube-multus\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546427811
Jul 17 03:28:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:24.468Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db56\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546437019
Jul 17 03:28:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:24.491Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_swap_limit_bytes\", asset=\"msh-tsgx09\", asset_id=\"55\", container=\"metrics-server\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546438586
Jul 17 03:28:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:24.504Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_threads\", asset=\"msh-tsgx09\", asset_id=\"55\", container=\"local-path-provisioner\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546427870
Jul 17 03:28:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:24.517Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_created\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.tsg-traffic-engine-numa-0.v1\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546440068
Jul 17 03:28:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:24.532Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_hwmon_temp_celsius\", asset=\"msh-tsgx09\", asset_id=\"55\", chip=\"pci0000:00_0000:00:18_3\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", sensor=\"temp5\", service=\"prometheus-node-exporter\"}" timestamp=1689546434790
Jul 17 03:28:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:24.613Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-a6af3228b30486917a2822a45867102e7100c8725685d2f1b68ab6917baebdfb-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689546438416
Jul 17 03:28:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:24.641Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_hwmon_temp_celsius\", asset=\"msh-tsgx09\", asset_id=\"55\", chip=\"pci0000:00_0000:00:19_3\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", sensor=\"temp9\", service=\"prometheus-node-exporter\"}" timestamp=1689546434790
Jul 17 03:28:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:24.659Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_info\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", job_name=\"helm-install-systemd-exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546431122
Jul 17 03:28:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:24.707Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_fifo_total\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"vethf1f0f7fc\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546434790
Jul 17 03:28:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:24.716Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-ens4f3np3.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689546438416
Jul 17 03:28:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:24.734Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-vethcf071709.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689546438416
Jul 17 03:28:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:24.757Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_metadata_resource_version\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"k3s-serving\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546440068
Jul 17 03:28:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:24.782Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_errs_total\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens4f2np2\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546434790
Jul 17 03:28:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:24.951Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2did-wwn\\\\x2d0x62cea7f0848263002c2c205714e9cdc4\\\\x2dpart4.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689546438416
Jul 17 03:28:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:24.968Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:00-0000:00:01.1-0000:01:00.0-infiniband-mlx5_0.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689546438416
Jul 17 03:28:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:25.026Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_free_bytes\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"shm\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/2cecd16a141d9d68b235173be7c3e8115692fc6e05b6fdb1d80145067687e261/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546440160
Jul 17 03:28:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:25.070Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_rate_bytes_per_second\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"mlx5_10\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546440160
Jul 17 03:28:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:25.326Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_info\", asset=\"pcap-tsgx12\", asset_id=\"117\", configmap=\"chart-content-mrzcpd-exporter\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546431122
Jul 17 03:28:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:25.392Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"tmpfs\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/data/var/lib/kubelet/pods/89567228-6ae4-4ed5-8117-7627ccbebb6d/volumes/kubernetes.io~projected/kube-api-access-bhmtx\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546440160
Jul 17 03:28:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:25.788Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"51\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546440160
Jul 17 03:28:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:26.368Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_commands_failed_calls_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cmd=\"get\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546443549
Jul 17 03:28:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:26.804Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Udp_Link_S2C_ALL\", app_name=\"sapp4\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", vsysId=\"1\"}" timestamp=1689546435421
Jul 17 03:28:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:27.243Z caller=dedupe.go:112 component=remote level=error remote_name=52db6f url=http://10.10.10.159:10090/nz-agent/prometheus/proxy/api/v1/write msg="non-recoverable error" count=1000 exemplarCount=0 err="server returned HTTP status 400 : out of order sample"
Jul 17 03:28:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:27.408Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_annotations\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"prometheus-kube-state-metrics-6b768c7bf8\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546435682
Jul 17 03:28:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:27.496Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-traefik-crd-h8g7g\", project=\"TSG-OS\", project_id=\"7\", reason=\"UnexpectedAdmissionError\", service=\"prometheus-kube-state-metrics\", uid=\"e303f019-5869-4dc2-afe6-c1107cceb13d\"}" timestamp=1689546438744
Jul 17 03:28:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:27.512Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"76\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546435810
Jul 17 03:28:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:27.539Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"dashboard-metrics-scraper\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Sync\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546430829
Jul 17 03:28:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:27.558Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"exporter\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Total\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546437189
Jul 17 03:28:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:27.572Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_status_succeeded\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", job_name=\"helm-install-cm-policy-local-cache\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546435682
Jul 17 03:28:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:27.663Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_swap_limit_bytes\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"dign-server-dns\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546444841
Jul 17 03:28:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:27.683Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_created\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-alertmanager-6df7f457b6-pwr5q\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"25f2e5d4-c32e-4636-bf64-2b5006d78aad\"}" timestamp=1689546435682
Jul 17 03:28:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:27.708Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"119\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546442882
Jul 17 03:28:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:28.013Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_created\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"kubernetes\", uid=\"1b75e3e3-39e2-4abb-b886-2f4e451df33b\"}" timestamp=1689546435682
Jul 17 03:28:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:28.020Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"63\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546442882
Jul 17 03:28:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:28.030Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_bytes_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth002839a7\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546442882
Jul 17 03:28:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:28.145Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"117\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546442882
Jul 17 03:28:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:28.154Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"40\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546442882
Jul 17 03:28:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:28.227Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_netstat_TcpExt_TCPTimeouts\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546442882
Jul 17 03:28:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:28.235Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_up\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth2e13d535\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546442882
Jul 17 03:28:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:28.332Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_commands_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cmd=\"select\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546439446
Jul 17 03:28:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:28.353Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"var-lib-kubelet-pods-848571bc\\\\x2da31e\\\\x2d44de\\\\x2da0f0\\\\x2dd172cf8f3bd0-volume\\\\x2dsubpaths-config\\\\x2dvolume-telegraf-2.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689546434114
Jul 17 03:28:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:28.438Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_restarts_total\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"helm\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-traefik-rdmft\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"3daa5d94-8d7c-4d71-883e-255619668328\"}" timestamp=1689546435682
Jul 17 03:28:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:28.493Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_cpu_seconds_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-pod0bdb24ec_e0f8_44e3_8d3f_a437820bc4e9.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"slice\"}" timestamp=1689546434114
Jul 17 03:28:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:28.521Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"56\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546442882
Jul 17 03:28:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:28.532Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"abrtd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"service\"}" timestamp=1689546434114
Jul 17 03:28:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:28.563Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_up_changes_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"br_dign_s\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546442882
Jul 17 03:28:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:28.593Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_compressed_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"vethd626638d\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546442882
Jul 17 03:28:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:28.676Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"process_start_time_seconds\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546439446
Jul 17 03:28:29 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:29.026Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"40\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546442882
Jul 17 03:28:29 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:29.170Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"exporter-merger\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", failure_type=\"pgfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"hierarchy\"}" timestamp=1689546432794
Jul 17 03:28:29 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:29.242Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"prometheus-alertmanager-configmap-reload\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"uninterruptible\"}" timestamp=1689546425678
Jul 17 03:28:29 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:29.285Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_device_id\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens4f3np3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546442882
Jul 17 03:28:29 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:29.303Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_limit_bytes\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"prometheus-alertmanager\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546444841
Jul 17 03:28:29 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:29.357Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_bytes_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens7f1np1\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546442882
Jul 17 03:28:29 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:29.591Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_created\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"systemd-exporter-7f549d9485-9zshn\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"163ef19a-c78d-47c5-82c4-3bf5914611f1\"}" timestamp=1689546435682
Jul 17 03:28:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:31.030Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_labels\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"prometheus-server-6ff784d758\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546444068
Jul 17 03:28:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:31.063Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_multicast_total\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"vetha0f01bb9\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546440939
Jul 17 03:28:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:31.081Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_info\", asset=\"msh-tsgx05\", asset_id=\"67\", cluster_ip=\"10.43.163.218\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"6b9c82a0-3b6a-4544-b0bf-573e3b61f230\"}" timestamp=1689546444068
Jul 17 03:28:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:31.083Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_commands_failed_calls_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cmd=\"incrby\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546450141
Jul 17 03:28:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:31.158Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_processes\", asset=\"msh-tsgx05\", asset_id=\"67\", container=\"kube-multus\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546420015
Jul 17 03:28:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:31.211Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_info\", asset=\"msh-tsgx05\", asset_id=\"67\", board_id=\"MT_0000000359\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_6\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", firmware_version=\"22.28.4000\", hca_type=\"MT4125\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546440939
Jul 17 03:28:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:31.252Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"msh-tsgx05\", asset_id=\"67\", condition=\"unknown\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", pod=\"tsg-traffic-engine-numa-1-firewall-6d869d4b4c-7pv8f\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"d8012c00-aaec-4f56-a35b-ad6f4d1ef590\"}" timestamp=1689546444068
Jul 17 03:28:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:31.314Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-4079ea2e\\\\x2d97aa\\\\x2d4a21\\\\x2d888a\\\\x2d7befd9a893de-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2d2pchh.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689546441020
Jul 17 03:28:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:31.327Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_free_bytes\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"shm\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/5cccc2781248b9b56b6d4fa749ec3224f3942836c778396419467980f2edcce6/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546440939
Jul 17 03:28:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:31.423Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"13\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546440939
Jul 17 03:28:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:31.523Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"msh-tsgx05\", asset_id=\"67\", container=\"prometheus-alertmanager\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"iowaiting\"}" timestamp=1689546434305
Jul 17 03:28:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:31.544Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_info\", address=\"ba:b0:d8:96:0e:81\", asset=\"twa-tsgx03\", asset_id=\"95\", broadcast=\"ff:ff:ff:ff:ff:ff\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth662ec9f2\", duplex=\"full\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operstate=\"up\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546439093
Jul 17 03:28:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:31.599Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_created\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_endpoint=\"kube-dns\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546444068
Jul 17 03:28:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:31.693Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-b292e28f3be6b425441bd4f2c116a86aedaa02f683430f67a4331c6179d18735.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"scope\"}" timestamp=1689546441020
Jul 17 03:28:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:31.758Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"126\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546440939
Jul 17 03:28:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:31.787Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_drop_total\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth8d362ef6\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546440939
Jul 17 03:28:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:31.847Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546440939
Jul 17 03:28:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:31.862Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_scrape_collector_duration_seconds\", asset=\"msh-tsgx05\", asset_id=\"67\", collector=\"tapestats\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546440939
Jul 17 03:28:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:31.881Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_fifo_total\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens7f0np0\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546440939
Jul 17 03:28:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:32.040Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"33\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546440939
Jul 17 03:28:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:32.061Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"tmpfs\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/var/lib/kubelet/pods/d78263bf-476b-4921-a83b-0fe401ff42c2/volumes/kubernetes.io~projected/kube-api-access-jcw8d\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546440939
Jul 17 03:28:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:32.092Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"87\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546440939
Jul 17 03:28:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:32.115Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"56\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546439093
Jul 17 03:28:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:32.142Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_cpu_user_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546446438
Jul 17 03:28:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:32.267Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-infiniband-umad7.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689546446907
Jul 17 03:28:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:32.504Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_owner\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", owner_is_controller=\"true\", owner_kind=\"Deployment\", owner_name=\"tsg-traffic-engine-numa-1-firewall\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-1-firewall-6b6bf4679c\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546439894
Jul 17 03:28:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:32.513Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_writes_total\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546435807
Jul 17 03:28:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:32.523Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_ips\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", ip=\"10.42.0.67\", ip_family=\"4\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-pushgateway-bd476bcf9-5mpjx\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"f2075fac-2ce7-4f3c-b94c-4c05e2de4805\"}" timestamp=1689546439894
Jul 17 03:28:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:32.537Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_limit_bytes\", asset=\"msh-tsgx05\", asset_id=\"67\", container=\"dign-client\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546435836
Jul 17 03:28:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:32.541Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-1242875f9ccc3d5ab62db93f916bd67c06da5b8e47f89d7f923c00b4c43c725a.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"scope\"}" timestamp=1689546447691
Jul 17 03:28:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:32.606Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"28\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546440605
Jul 17 03:28:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:32.608Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"msh-tsgx05\", asset_id=\"67\", condition=\"Progressing\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"dashboard-metrics-scraper\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"false\"}" timestamp=1689546444068
Jul 17 03:28:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:32.647Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_created\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"dashboard-metrics-scraper-5b94897dc9\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546444068
Jul 17 03:28:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:32.663Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda5\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", fstype=\"ext4\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/data/var/lib/kubelet/pods/506c9da6-9ad1-4f6d-afc6-e4606512a111/volume-subpaths/config-volume/telegraf-security/1\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546440791
Jul 17 03:28:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:32.672Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas_available\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"dashboard-metrics-scraper\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546439894
Jul 17 03:28:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:32.679Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_speed_bytes\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"eno2\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546439093
Jul 17 03:28:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:32.695Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_type\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"chart-values-configmap-reloader\", service=\"prometheus-kube-state-metrics\", type=\"Opaque\"}" timestamp=1689546439894
Jul 17 03:28:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:32.695Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_packets_dropped_total\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"cadvisor\", interface=\"eth0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689546430196
Jul 17 03:28:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:32.701Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-sda4.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689546442644
Jul 17 03:28:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:32.712Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"54\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546440939
Jul 17 03:28:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:32.718Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_init_container_resource_requests\", asset=\"twa-tsgx05\", asset_id=\"99\", container=\"install-multus-binary\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", node=\"localhost\", nz_agent_id=\"3\", pod=\"kube-multus-ds-vmk6p\", project=\"TSG-OS\", project_id=\"7\", resource=\"memory\", service=\"prometheus-kube-state-metrics\", uid=\"e3971f33-46bb-450e-a67b-8e9e2f2ec265\", unit=\"byte\"}" timestamp=1689546439894
Jul 17 03:28:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:32.734Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"43\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546440605
Jul 17 03:28:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:32.738Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_vl15_dropped_total\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_4\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546440605
Jul 17 03:28:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:32.752Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx05\", asset_id=\"67\", container=\"configmap-reloader\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Discard\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546423019
Jul 17 03:28:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:32.761Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"39\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546440605
Jul 17 03:28:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:32.792Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"process_max_fds\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546439093
Jul 17 03:28:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:32.818Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"svclb-traefik-acf36120-2ckwj\", project=\"TSG-OS\", project_id=\"7\", reason=\"NodeAffinity\", service=\"prometheus-kube-state-metrics\", uid=\"14f66a9f-85bf-4104-ac3f-c15dda542ad5\"}" timestamp=1689546450436
Jul 17 03:28:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:32.835Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cryptsetup.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"target\"}" timestamp=1689546446907
Jul 17 03:28:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:32.855Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas_unavailable\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"mrzcpd-exporter\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546439894
Jul 17 03:28:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:32.870Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_changes_total\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth28573ba4\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546440605
Jul 17 03:28:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:32.885Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"rshim.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689546447691
Jul 17 03:28:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:32.925Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"fcoe.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"service\"}" timestamp=1689546441020
Jul 17 03:28:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:32.981Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"17\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546439093
Jul 17 03:28:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:33.006Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_user_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", container=\"prometheus-alertmanager\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546429256
Jul 17 03:28:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:33.026Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_writes_merged_total\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda2\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546437038
Jul 17 03:28:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:33.058Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"77\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546439093
Jul 17 03:28:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:33.069Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_last_terminated_reason\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"configmap-reloader\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"configmap-reloader-6c55bbc49f-mkbdr\", project=\"TSG-OS\", project_id=\"7\", reason=\"Unknown\", service=\"prometheus-kube-state-metrics\", uid=\"69597224-3b13-4fb1-98ff-fc4bdacb87f6\"}" timestamp=1689546439276
Jul 17 03:28:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:33.074Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"88\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546439093
Jul 17 03:28:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:33.081Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_netstat_Udp_RcvbufErrors\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546440791
Jul 17 03:28:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:33.130Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_avail_bytes\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"shm\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/6a9cd9c06f7baec3789fea8eb387ebed130e7bbba376657e2a03643e0365e3d7/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546439093
Jul 17 03:28:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:33.131Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_inodes_free\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/run\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546434922
Jul 17 03:28:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:33.142Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"tmpfs\", name=\"tmp.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689546446907
Jul 17 03:28:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:33.149Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", failure_type=\"pgfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"hierarchy\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689546433084
Jul 17 03:28:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:33.153Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dracut-pre-pivot.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"oneshot\", type=\"service\"}" timestamp=1689546446907
Jul 17 03:28:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:33.157Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_mtu_bytes\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth15723f66\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546440791
Jul 17 03:28:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:33.169Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"81\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546440791
Jul 17 03:28:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:33.191Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx05\", asset_id=\"99\", container=\"dign-client\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Total\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546424459
Jul 17 03:28:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:33.198Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_annotations\", asset=\"twa-tsgx03\", asset_id=\"95\", configmap=\"chart-content-mrzcpd-exporter\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546450436
Jul 17 03:28:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:33.222Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_files_free\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"shm\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/21cbe6f5176dae7827895c0bcee3fe7ab129604ae49f1dc81fff10e80b6f9358/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546439093
Jul 17 03:28:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:33.238Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_dormant\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens4f1np1\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546439093
Jul 17 03:28:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:33.239Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"twa-tsgx05\", asset_id=\"99\", container=\"dashboard-metrics-scraper\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"cadvisor\", failure_type=\"pgfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"hierarchy\"}" timestamp=1689546426355
Jul 17 03:28:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:33.258Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_ready\", asset=\"twa-tsgx05\", asset_id=\"99\", container=\"metrics\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"cm-policy-local-cache-redis-master-69967b5879-vlgl8\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"e8ef10b6-3745-4720-b3f4-f80080df3d45\"}" timestamp=1689546439894
Jul 17 03:28:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:33.285Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_compressed_total\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens6f1np1\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546440791
Jul 17 03:28:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:33.379Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_max_usage_bytes\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"redis\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546438868
Jul 17 03:28:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:33.392Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_packets_total\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", interface=\"br_dign_c\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546439464
Jul 17 03:28:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:33.395Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db219\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546449260
Jul 17 03:28:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:33.396Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_working_set_bytes\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689546431201
Jul 17 03:28:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:33.410Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_created\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"kubernetes-dashboard-certs\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546439276
Jul 17 03:28:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:33.517Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_user_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546438374
Jul 17 03:28:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:33.534Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_bytes_total\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth52705744\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546440605
Jul 17 03:28:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:33.548Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_cpu_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-podfaaa93b7_eae4_4c14_96b8_0ca5e8ca7cad.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"slice\"}" timestamp=1689546447691
Jul 17 03:28:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:33.582Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_rss\", asset=\"twa-tsgx05\", asset_id=\"99\", container=\"prometheus-alertmanager\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546437839
Jul 17 03:28:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:33.603Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"msh-tsgx11\", asset_id=\"111\", condition=\"Progressing\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"systemd-exporter\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"false\"}" timestamp=1689546439276
Jul 17 03:28:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:33.606Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_created\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_endpoint=\"prometheus-alertmanager\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546439276
Jul 17 03:28:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:33.622Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_info\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.ipmi-exporter.v1\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546439276
Jul 17 03:28:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:33.638Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_errs_total\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"eno33np0\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546440791
Jul 17 03:28:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:33.700Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"42\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546440791
Jul 17 03:28:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:33.704Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_scrape_collector_success\", asset=\"twa-tsgx05\", asset_id=\"99\", collector=\"bonding\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546440605
Jul 17 03:28:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:33.796Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas_ready\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"coredns\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546439894
Jul 17 03:28:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:33.801Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"pcap-tsgx05\", asset_id=\"77\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda5\", endpoint=\"TSG-OS-pcap-tsgx05\", endpoint_id=\"471\", exported_job=\"exporter\", fstype=\"ext4\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/var/lib/kubelet/pods/f198417b-e776-4e41-b3ac-723f6d33c393/volume-subpaths/telegraf/telegraf-security/0\", nz_agent_id=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546444387
Jul 17 03:28:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:33.901Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"nfs-utils.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"service\"}" timestamp=1689546447691
Jul 17 03:28:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:33.936Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_info\", asset=\"twa-tsgx05\", asset_id=\"99\", created_by_kind=\"Job\", created_by_name=\"helm-install-prometheus\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", host_ip=\"10.10.30.165\", host_network=\"false\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", node=\"localhost\", nz_agent_id=\"3\", pod=\"helm-install-prometheus-jgzhf\", pod_ip=\"10.42.0.60\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"f2f577b5-5389-48c3-9e35-cfcac2fb3148\"}" timestamp=1689546439894
Jul 17 03:28:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:33.982Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Failed\", pod=\"metrics-server-54dc485875-pdn8r\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"3538cc3c-71c8-48b7-8345-977c5f681b51\"}" timestamp=1689546439894
Jul 17 03:28:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:34.001Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_commands_duration_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cmd=\"del\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546449260
Jul 17 03:28:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:34.114Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_multicast_total\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethf1fa10d9\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546440605
Jul 17 03:28:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:34.138Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_write_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/sys/fs/cgroup\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546436889
Jul 17 03:28:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:34.226Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas_unavailable\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"cm-policy-local-cache-redis-master\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546439894
Jul 17 03:28:36 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:36.137Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kafka_server_BrokerTopicMetrics_OneMinuteRate\", asset=\"pcap-dellr740-dt002\", asset_id=\"33\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-Kafka-pcap-dellr740-dt002\", endpoint_id=\"101\", module=\"DC-Kafka\", module_id=\"5\", name=\"MessagesInPerSec\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", topic=\"DOS-EVENT\"}" timestamp=1689546455787
Jul 17 03:28:36 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:36.236Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kafka_network_RequestMetrics_Errors_total\", asset=\"pcap-dellr740-dt002\", asset_id=\"33\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-Kafka-pcap-dellr740-dt002\", endpoint_id=\"101\", error=\"REBALANCE_IN_PROGRESS\", module=\"DC-Kafka\", module_id=\"5\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", request=\"Heartbeat\"}" timestamp=1689546455787
Jul 17 03:28:39 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:39.265Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"twa-dellr740-dt003\", asset_id=\"41\", cpu=\"35\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"OLAP-Node-Exporter-twa-dellr740-dt003\", endpoint_id=\"261\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"hbase_master,clickhouse_data,kafka,gohangout,zookeeper,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546458473
Jul 17 03:28:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:40.383Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"114\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546451647
Jul 17 03:28:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:40.400Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-0bcfe390606ef1e4f104df6119a365ccc0e5fcf9670d1445a91965f46fa871a5-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689546455764
Jul 17 03:28:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:40.416Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sshd-keygen@ecdsa.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"service\"}" timestamp=1689546455764
Jul 17 03:28:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:40.467Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_bytes_total\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth219b0248\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546451647
Jul 17 03:28:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:40.479Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_queue_length\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens2f1np1\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546451647
Jul 17 03:28:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:40.532Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"twa-tsgx04\", asset_id=\"97\", container=\"kube-multus\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"stopped\"}" timestamp=1689546444963
Jul 17 03:28:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:40.541Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_annotations\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"ipmi-exporter-6f8464dd94-c5t7s\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"87f90b18-7e5c-4eda-916d-6770523a8590\"}" timestamp=1689546460034
Jul 17 03:28:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:40.575Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"gpm.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"forking\", type=\"service\"}" timestamp=1689546455764
Jul 17 03:28:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:40.662Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_gc_duration_seconds_sum\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546453827
Jul 17 03:28:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:40.676Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_metadata_resource_version\", asset=\"twa-tsgx04\", asset_id=\"97\", configmap=\"certstore-tsg-traffic-engine-numa-1\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546460034
Jul 17 03:28:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:40.717Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"8\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546451647
Jul 17 03:28:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:40.837Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_system_table_bucket_meta_metric_compactionInputFileCount_90th_percentile\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546459757
Jul 17 03:28:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:40.867Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2dlabel-ONIE\\\\x2dBOOT.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689546455764
Jul 17 03:28:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:40.984Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx04\", asset_id=\"97\", container=\"dign-server-web\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Total\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546436601
Jul 17 03:28:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:41.070Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_storeCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"proxy_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"9e185aa8f79610d5667059ec3c98abf3\", table=\"index_time_proxy_hos_bucket\", type=\"index\"}" timestamp=1689546459757
Jul 17 03:28:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:41.139Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_10_194_1_metric_compactionInputFileCount_median\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546459757
Jul 17 03:28:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:41.285Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"bond0\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546451647
Jul 17 03:28:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:41.305Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_numFilesCompactedCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"default\", project=\"OLAP\", project_id=\"3\", region=\"f598627811b25269880e86a179afcf08\", table=\"index_filename_default\", type=\"index\"}" timestamp=1689546459757
Jul 17 03:28:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:41.334Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_10_194_1_metric_maxStoreFileAge\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546459757
Jul 17 03:28:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:41.338Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_CHILD_LINK_metric_incrementTime_75th_percentile\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546459757
Jul 17 03:28:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:41.364Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_PauseTimeWithoutGc_num_ops\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Server\"}" timestamp=1689546459757
Jul 17 03:28:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:41.618Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"27\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546451647
Jul 17 03:28:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:41.695Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_files\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"shm\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/0bcfe390606ef1e4f104df6119a365ccc0e5fcf9670d1445a91965f46fa871a5/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546451647
Jul 17 03:28:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:41.737Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx04\", asset_id=\"97\", container=\"dign-client\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Total\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546442891
Jul 17 03:28:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:41.895Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_readRequestCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"proxy_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"c01229b821e0a41e45bab953568a5e51\", table=\"index_partfile_proxy_hos_bucket\", type=\"index\"}" timestamp=1689546459757
Jul 17 03:28:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:41.921Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_hbase_table_rsgroup_metric_putBatchTime_75th_percentile\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546459757
Jul 17 03:28:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:41.938Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_created\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_endpoint=\"prometheus-alertmanager\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546460034
Jul 17 03:28:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:41.941Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_appendCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"assessment_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"366e45393482e2921e80c3b6f418d5f3\", table=\"index_time_assessment_hos_bucket\", type=\"index\"}" timestamp=1689546459757
Jul 17 03:28:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:41.995Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_memStoreSize\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"firewall_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"a4729c2c59157f480f8c0f05b3fc7cb3\", table=\"index_partfile_firewall_hos_bucket\", type=\"index\"}" timestamp=1689546459757
Jul 17 03:28:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:42.018Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_CATALOG_metric_tableWriteQueryPerSecond_15min_rate\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546459757
Jul 17 03:28:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:42.044Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_LOG_metric_tableReadQueryPerSecond_5min_rate\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546459757
Jul 17 03:28:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:42.063Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_avgStoreFileAge\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"troubleshooting_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"f942b1b1098244a4ec1b2578b462bd41\", table=\"index_time_troubleshooting_hos_bucket\", type=\"index\"}" timestamp=1689546459757
Jul 17 03:28:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:42.069Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_10_194_1_metric_majorCompactionTime_num_ops\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546459757
Jul 17 03:28:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:42.102Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_created\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"chart-values-prometheus\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546460034
Jul 17 03:28:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:42.124Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"0\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546451647
Jul 17 03:28:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:42.176Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"twa-tsgx04\", asset_id=\"97\", condition=\"unknown\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-tsg-traffic-engine-numa-0-qwtq5\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"8bcaaf2d-628e-4f10-872c-7a32ecf2946e\"}" timestamp=1689546460034
Jul 17 03:28:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:42.407Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_storeFileSize\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"default\", project=\"OLAP\", project_id=\"3\", region=\"8556e3172d7fcf2817d95c152900f994\", table=\"index_time_default\", type=\"index\"}" timestamp=1689546459757
Jul 17 03:28:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:42.488Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"usr_tx_drop_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", dev=\"ens1f1np1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"mrzcpd-exporter\"}" timestamp=1689546459613
Jul 17 03:28:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:42.546Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_nf_conntrack_stat_early_drop\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546456242
Jul 17 03:28:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:42.573Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sssd-kcm.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689546460671
Jul 17 03:28:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:42.574Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_bytes_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"cadvisor\", interface=\"ens2f1np1\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546457548
Jul 17 03:28:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:42.590Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"tsg_master_activing\", app_name=\"tsg_master\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"traffic-engine\", line_name=\"success\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", table_name=\"ctrl_stat\", vsysId=\"1\"}" timestamp=1689546449178
Jul 17 03:28:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:42.611Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_cpu_quota\", asset=\"pcap-tsgx01\", asset_id=\"71\", container=\"kube-multus\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546460951
Jul 17 03:28:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:42.622Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", cpu=\"37\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546456242
Jul 17 03:28:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:42.625Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_drop_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth585262fb\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546456242
Jul 17 03:28:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:42.645Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"tsg_master_success_log_s\", app_name=\"tsg_master\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"traffic-engine\", line_name=\"SECURITY-EVENT\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", table_name=\"sendlog\", vsysId=\"1\"}" timestamp=1689546447581
Jul 17 03:28:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:42.666Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_10_194_1_metric_compactionInputFileCount_min\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546459757
Jul 17 03:28:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:42.744Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_tsg_galaxy_table_relation_account_framedip_metric_putTime_99_9th_percentile\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546459757
Jul 17 03:28:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:42.762Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_appendCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"proxy_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"c01229b821e0a41e45bab953568a5e51\", table=\"index_partfile_proxy_hos_bucket\", type=\"index\"}" timestamp=1689546459757
Jul 17 03:28:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:43.202Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_io_time_seconds_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/run\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546457548
Jul 17 03:28:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:43.220Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_multicast_packets_received_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"mlx5_11\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546456242
Jul 17 03:28:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:43.311Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_frequency_min_hertz\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", cpu=\"41\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Gohangout-msh-dellr740-dt002\", endpoint_id=\"167\", module=\"DC-Gohangout\", module_id=\"35\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546462625
Jul 17 03:28:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:43.400Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_drop_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens6f0np0\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546456242
Jul 17 03:28:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:43.424Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", cpu=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546456242
Jul 17 03:28:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:43.440Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", cpu=\"41\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546456242
Jul 17 03:28:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:43.451Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_writes_bytes_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", container=\"firewall\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689546452721
Jul 17 03:28:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:43.470Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", cpu=\"66\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546456242
Jul 17 03:28:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:43.543Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-journal-flush.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"oneshot\", type=\"service\"}" timestamp=1689546460671
Jul 17 03:28:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:43.575Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth017375a5\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546456242
Jul 17 03:28:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:43.667Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_errors_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"cadvisor\", interface=\"ens3f0np0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546457548
Jul 17 03:28:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:43.733Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", cpu=\"55\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546456242
Jul 17 03:28:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:43.795Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_load_average_10s\", asset=\"pcap-tsgx01\", asset_id=\"71\", container=\"exporter-merger\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546455165
Jul 17 03:28:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:43.817Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", cpu=\"14\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546456242
Jul 17 03:28:46 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:46.150Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_frees_total\", asset=\"msh-kvm-cm001\", asset_id=\"3\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"CM-Node-Exporter-msh-kvm-cm001\", endpoint_id=\"391\", module=\"CM-Node-Exporter\", module_id=\"75\", nz_agent_id=\"3\", parent_asset=\"msh-dellr740-pve001\", parent_asset_id=\"1\", project=\"Central-Management\", project_id=\"5\"}" timestamp=1689546464235
Jul 17 03:28:48 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:48.719Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_filteredReadRequestCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"a0a5e2a765062263dffb9516833c9adf\", table=\"troubleshooting_hos_bucket\", type=\"data\"}" timestamp=1689546467438
Jul 17 03:28:48 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:48.916Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_default_metric_tableReadQueryPerSecond_15min_rate\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546467438
Jul 17 03:28:48 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:48.949Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_session_record_hos_bucket_metric_flushTime_num_ops\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546467438
Jul 17 03:28:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:49.256Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_2_metric_scanSize_max\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546467438
Jul 17 03:28:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:49.280Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_maxStoreFileAge\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"troubleshooting_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"e5c020dc014f5aec69496addff336af7\", table=\"index_partfile_troubleshooting_hos_bucket\", type=\"index\"}" timestamp=1689546467438
Jul 17 03:28:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:49.325Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_incrementCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"default\", project=\"OLAP\", project_id=\"3\", region=\"d83cc1aea06dd989e81e7b9e4481ca14\", table=\"index_filename_default\", type=\"index\"}" timestamp=1689546467438
Jul 17 03:28:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:49.349Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_193_1_metric_tableWriteQueryPerSecond_mean_rate\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546467438
Jul 17 03:28:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:49.533Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_physical_state_id\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546461797
Jul 17 03:28:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:49.567Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_virtual_memory_bytes\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"packagekit.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689546461057
Jul 17 03:28:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:49.621Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_avgStoreFileAge\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"rtp_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"4ffb0332e3760623193f0865a9728ddf\", table=\"index_partfile_rtp_hos_bucket\", type=\"index\"}" timestamp=1689546467438
Jul 17 03:28:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:49.631Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"tmpfs\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/data/var/lib/kubelet/pods/71f6269b-5154-46cd-b538-223c28b8f982/volumes/kubernetes.io~projected/kube-api-access-4mwcb\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546461797
Jul 17 03:28:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:49.641Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-10f48067c51ea52a333a52d4f0a18626e100ee9028a1819014690990a9682620-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689546461057
Jul 17 03:28:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:49.682Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"28\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546461797
Jul 17 03:28:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:49.703Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx10\", asset_id=\"109\", cpu=\"106\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546461797
Jul 17 03:28:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:49.724Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"coredns-766944d45b-dstvr\", project=\"TSG-OS\", project_id=\"7\", reason=\"Shutdown\", service=\"prometheus-kube-state-metrics\", uid=\"b52de515-fecd-439f-b56f-e8953f782519\"}" timestamp=1689546459668
Jul 17 03:28:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:49.815Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_io_time_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/var/lib/kubelet/pods/2cfe23ce-7e71-44dc-8e19-b936dd7c7f8c/volumes/kubernetes.io~projected/kube-api-access-msvmx\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546461880
Jul 17 03:28:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:49.859Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx10\", asset_id=\"109\", cpu=\"111\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546461797
Jul 17 03:28:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:49.866Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_memory_SUnreclaim_bytes\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546466709
Jul 17 03:28:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:49.872Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_complete\", asset=\"twa-tsgx08\", asset_id=\"105\", condition=\"unknown\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", job_name=\"helm-delete-tsg-traffic-engine-numa-1\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546461568
Jul 17 03:28:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:49.888Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_scrape_collector_success\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", collector=\"rapl\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-olap003\", endpoint_id=\"235\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"yarn_resourcemanager,zookeeper,nacos,kafka,spark,druid_data,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546469714
Jul 17 03:28:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:49.902Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_packets_total\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", interface=\"eno36np3\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546457732
Jul 17 03:28:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:49.903Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"tsg_master_add\", app_name=\"tsg_master\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"traffic-engine\", line_name=\"tunnel_endpoint\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", table_name=\"sync_exdata\", vsysId=\"1\"}" timestamp=1689546462447
Jul 17 03:28:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:49.920Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"process_resident_memory_bytes\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546467556
Jul 17 03:28:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:49.921Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", cpu=\"1\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-olap003\", endpoint_id=\"235\", mode=\"nice\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"yarn_resourcemanager,zookeeper,nacos,kafka,spark,druid_data,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546469714
Jul 17 03:28:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:49.931Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"twa-tsgx10\", asset_id=\"109\", cpu=\"87\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546461797
Jul 17 03:28:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:49.940Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"twa-tsgx08\", asset_id=\"105\", container=\"firewall\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", state=\"stopped\"}" timestamp=1689546463761
Jul 17 03:28:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:49.941Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", cpu=\"14\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-olap003\", endpoint_id=\"235\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"yarn_resourcemanager,zookeeper,nacos,kafka,spark,druid_data,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546469714
Jul 17 03:28:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:49.947Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"twa-tsgx10\", asset_id=\"109\", condition=\"Progressing\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"metrics-server\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"unknown\"}" timestamp=1689546459668
Jul 17 03:28:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:49.956Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_storeCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", namespace=\"SYSTEM\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"a289e6c6c4fb50405ce3745629bf659b\", table=\"LOG\", type=\"data\"}" timestamp=1689546467438
Jul 17 03:28:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:49.960Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_protocol_type\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"virtio_dign_c\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546466709
Jul 17 03:28:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:49.972Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_flushesQueuedCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"4742e1c0c5331a4df4bdd23109c9d4a9\", table=\"troubleshooting_hos_bucket\", type=\"data\"}" timestamp=1689546467438
Jul 17 03:28:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:49.982Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_persistentvolume_status_phase\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", persistentvolume=\"pvc-c08dcd6c-f603-4169-ae73-c85cac6a245f\", phase=\"Pending\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546461568
Jul 17 03:28:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:50.020Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_restarts_total\", asset=\"twa-tsgx08\", asset_id=\"105\", container=\"mrapm-device\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"mrzcpd-exporter-5cf7f967cc-b62w2\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"c4ec4f52-224c-4f5b-9432-db80b6600279\"}" timestamp=1689546461568
Jul 17 03:28:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:50.021Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-infiniband-umad2.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689546466554
Jul 17 03:28:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:50.158Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_reads_merged_total\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda2\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546453521
Jul 17 03:28:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:50.177Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"twa-tsgx10\", asset_id=\"109\", cpu=\"82\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546461797
Jul 17 03:28:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:50.187Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"dashboard-metrics-scraper\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Total\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546447125
Jul 17 03:28:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:50.200Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db252\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546466625
Jul 17 03:28:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:50.205Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_scaling_frequency_hertz\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"43\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546466709
Jul 17 03:28:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:50.213Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", cpu=\"0\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-olap003\", endpoint_id=\"235\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"yarn_resourcemanager,zookeeper,nacos,kafka,spark,druid_data,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546469714
Jul 17 03:28:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:50.221Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"81\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546466709
Jul 17 03:28:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:50.344Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", cpu=\"3\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-olap003\", endpoint_id=\"235\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"yarn_resourcemanager,zookeeper,nacos,kafka,spark,druid_data,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546469714
Jul 17 03:28:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:50.346Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx08\", asset_id=\"105\", container=\"dign-server-web\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Async\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546460713
Jul 17 03:28:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:50.390Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"83\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546466709
Jul 17 03:28:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:50.489Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"dign-server-dns\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"running\"}" timestamp=1689546452043
Jul 17 03:28:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:50.519Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_spec_paused\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"prometheus-pushgateway\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546459668
Jul 17 03:28:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:50.523Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_metadata_generation\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"local-path-provisioner-957fdf8bc\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546459668
Jul 17 03:28:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:50.546Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"70\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546461797
Jul 17 03:28:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:50.633Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"22\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546466709
Jul 17 03:28:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:50.647Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"4\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546466709
Jul 17 03:28:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:50.692Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db242\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546467556
Jul 17 03:28:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:50.776Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_ready_replicas\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"prometheus-alertmanager-6df7f457b6\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546459668
Jul 17 03:28:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:50.832Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_info\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"mrapm-device\", container_id=\"containerd://7554ff525f8f143bb91e9ee7736d0a155c1a23948b078bd5970ca5988327ea9b\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", image=\"registry.gdnt-cloud.website/mrzcpd-exporter:1.0.1\", image_id=\"sha256:b1a7adfb61be24490a1321c37405d86204b78b1ee251ed959d48dc357310fb1c\", image_spec=\"registry.gdnt-cloud.website/mrzcpd-exporter:1.0.1\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"mrzcpd-exporter-5cf7f967cc-mp8pv\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"71f6269b-5154-46cd-b538-223c28b8f982\"}" timestamp=1689546459668
Jul 17 03:28:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:51.074Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_bytes_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth9afe06eb\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546466709
Jul 17 03:28:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:51.157Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_exporter_build_info\", asset=\"twa-tsgx08\", asset_id=\"105\", build_date=\"2023-04-09-14:54:33\", commit_sha=\"b5e02003cea4b73054abe29433c264dec16cc1f0\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", golang_version=\"go1.20.2\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\", version=\"v1.50.0\"}" timestamp=1689546467556
Jul 17 03:28:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:51.908Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"cache_xa_rollbacks_total\", application=\"bifang_api\", asset=\"pcap-kvm-cm001\", asset_id=\"45\", cache=\"APPLICATION\", cacheManager=\"cacheManager\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"CM-API-pcap-kvm-cm001\", endpoint_id=\"285\", module=\"CM-API\", module_id=\"55\", name=\"APPLICATION\", nz_agent_id=\"3\", parent_asset=\"pcap-dellr740-pve001\", parent_asset_id=\"43\", project=\"Central-Management\", project_id=\"5\", result=\"success\"}" timestamp=1689546470876
Jul 17 03:28:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:52.593Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_state_started\", asset=\"pcap-tsgx11\", asset_id=\"115\", container=\"telegraf-security\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", pod=\"tsg-traffic-engine-numa-0-firewall-7c999f565f-xq5dg\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"93005333-2bbb-4ce3-a6b6-a5b632269832\"}" timestamp=1689546464478
Jul 17 03:28:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:52.608Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"45\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546469337
Jul 17 03:28:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:52.687Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_compressed_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"vethf21ae711\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546469337
Jul 17 03:28:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:52.781Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", cpu=\"81\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546469337
Jul 17 03:28:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:52.889Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", cpu=\"10\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546469337
Jul 17 03:28:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:52.890Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_flags\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"eno34np1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546469337
Jul 17 03:28:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:52.947Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_errs_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"eno1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546469337
Jul 17 03:28:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:53.095Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_type\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.cm-policy-local-cache.v1\", service=\"prometheus-kube-state-metrics\", type=\"helm.sh/release.v1\"}" timestamp=1689546464478
Jul 17 03:28:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:53.184Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"metrics-server-54dc485875-pqvd7\", project=\"TSG-OS\", project_id=\"7\", reason=\"NodeAffinity\", service=\"prometheus-kube-state-metrics\", uid=\"aed8aa53-e603-467c-8ed7-d385c606efb7\"}" timestamp=1689546464478
Jul 17 03:28:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:53.216Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_user_seconds_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", container=\"exporter\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546467580
Jul 17 03:28:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:53.220Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_gc_duration_seconds_sum\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"ipmi-exporter\"}" timestamp=1689546471652
Jul 17 03:28:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:53.241Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_virtual_memory_bytes\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"tsg-clixon-backend.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689546465496
Jul 17 03:28:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:53.255Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.grpc.v1.cri-sandboxes-a9011f6e3a4566a7feaeb28a5ed6eda14c5dedfa3db492ed9e31d50ef6fb4a69-shm.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689546465496
Jul 17 03:28:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:53.281Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_daemonset_status_number_available\", asset=\"msh-tsgx12\", asset_id=\"113\", daemonset=\"svclb-traefik-ad22eefe\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546468663
Jul 17 03:28:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:53.290Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-tsg-traffic-engine-numa-0-rftjq\", project=\"TSG-OS\", project_id=\"7\", reason=\"Shutdown\", service=\"prometheus-kube-state-metrics\", uid=\"55e6f7ec-a65b-4932-86c3-e88f9111393a\"}" timestamp=1689546468950
Jul 17 03:28:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:53.306Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_receive_remote_physical_errors_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"mlx5_6\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546469337
Jul 17 03:28:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:53.310Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_carrier_total\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"vethe1c8b7ca\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546468328
Jul 17 03:28:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:53.320Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db216\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546459367
Jul 17 03:28:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:53.328Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_bytes_total\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens4f0np0\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546468328
Jul 17 03:28:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:53.342Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_annotations\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", job_name=\"helm-install-prometheus\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546468950
Jul 17 03:28:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:53.353Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_down_changes_total\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"vethab8afd71\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546468328
Jul 17 03:28:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:53.359Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"msh-tsgx12\", asset_id=\"113\", condition=\"false\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-mrzcpd-exporter-dnnph\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"cfb99589-b1d8-4b8b-b2c3-d91a1a70fd6b\"}" timestamp=1689546468663
Jul 17 03:28:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:53.369Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"dign-server-ml4qh\", project=\"TSG-OS\", project_id=\"7\", reason=\"NodeAffinity\", service=\"prometheus-kube-state-metrics\", uid=\"2ed77d82-be58-47ef-9e9a-55c3cc7c3703\"}" timestamp=1689546468663
Jul 17 03:28:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:53.374Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_virtual_memory_bytes\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"getty@tty1.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689546471179
Jul 17 03:28:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:53.410Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", container=\"telegraf\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", state=\"sleeping\"}" timestamp=1689546460388
Jul 17 03:28:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:53.506Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_annotations\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"shaping-announce-port-tsg-traffic-engine-numa-1\", uid=\"4e95493c-5d84-4fe4-ae91-7935bee79e2c\"}" timestamp=1689546468663
Jul 17 03:28:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:53.544Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"msh-tsgx12\", asset_id=\"113\", cpu=\"117\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546463391
Jul 17 03:28:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:53.570Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_virtual_memory_bytes\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"snmpd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689546471179
Jul 17 03:28:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:53.639Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:60-0000:60:03.1-0000:63:00.0-net-eno33np0.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689546471179
Jul 17 03:28:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:53.772Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_spec_paused\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"metrics-server\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546462058
Jul 17 03:28:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:53.803Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Tcp_Bypass_Bit\", app_name=\"sapp4\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", vsysId=\"1\"}" timestamp=1689546462596
Jul 17 03:28:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:53.823Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db188\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546470108
Jul 17 03:28:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:53.849Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_swap\", asset=\"msh-tsgx08\", asset_id=\"65\", container=\"exporter\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546449256
Jul 17 03:28:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:53.869Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx12\", asset_id=\"113\", cpu=\"64\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546463391
Jul 17 03:28:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:53.880Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"nss-user-lookup.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"target\"}" timestamp=1689546465496
Jul 17 03:28:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:53.892Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"mdmonitor.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"service\"}" timestamp=1689546465496
Jul 17 03:28:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:53.994Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db252\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546470108
Jul 17 03:28:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:54.068Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_hwmon_temp_max_celsius\", asset=\"pcap-dellr740-dt002\", asset_id=\"33\", chip=\"platform_coretemp_1\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"NC-Gohangout-pcap-dellr740-dt002\", endpoint_id=\"173\", module=\"DC-Gohangout\", module_id=\"35\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sensor=\"temp1\"}" timestamp=1689546472666
Jul 17 03:28:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:54.072Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-netns-cni\\\\x2d4b6413e4\\\\x2da64b\\\\x2dfb11\\\\x2d34e6\\\\x2d245702c9991a.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689546460884
Jul 17 03:28:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:54.219Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"msh-tsgx12\", asset_id=\"113\", cpu=\"43\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546463391
Jul 17 03:28:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:54.283Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_dormant\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"cnibr0\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546459409
Jul 17 03:28:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:54.303Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_up\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"vethfd1158fa\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546459409
Jul 17 03:28:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:54.393Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"getty-pre.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"target\"}" timestamp=1689546460884
Jul 17 03:28:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:54.495Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_readonly\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"shm\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/2b106e9725417e4ba18199897aeb77289babcefc817de289178d2c87208a7198/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546468328
Jul 17 03:28:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:54.563Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_observed_generation\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"coredns-749bdddcdf\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546468950
Jul 17 03:28:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:54.734Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_ready\", asset=\"msh-tsgx12\", asset_id=\"113\", container=\"lb-tcp-443\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"svclb-traefik-ad22eefe-x9glh\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"7c7040d2-bb49-4f76-ac55-4ff20b58ef7e\"}" timestamp=1689546468663
Jul 17 03:28:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:54.876Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"122\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546463391
Jul 17 03:28:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:55.040Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", container=\"dign-server-web\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"cadvisor\", failure_type=\"pgfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"container\"}" timestamp=1689546453330
Jul 17 03:28:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:55.066Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link_mode\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens3f0np0\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546459409
Jul 17 03:28:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:55.074Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"msh-tsgx12\", asset_id=\"113\", container=\"metrics-server\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"iowaiting\"}" timestamp=1689546456627
Jul 17 03:28:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:55.087Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx08\", asset_id=\"65\", cpu=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546468328
Jul 17 03:28:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:55.107Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_hwmon_chip_names\", asset=\"msh-tsgx08\", asset_id=\"65\", chip=\"0000:80:01_2_0000:82:00_2\", chip_name=\"bnxt_en\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546468328
Jul 17 03:28:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:55.168Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-6d346642d292a4caed2d720d4b15a475683fe125ab624124ff968b5fb279c9a9-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689546471179
Jul 17 03:28:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:55.185Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda5\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", fstype=\"ext4\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/var/lib/kubelet/pods/8ba148c1-f25f-4fa4-82b3-03b9d5a8ace6/volume-subpaths/telegraf/telegraf-security/0\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546463391
Jul 17 03:28:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:55.187Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-ask-password-wall.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689546471179
Jul 17 03:28:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:55.220Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_files_free\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"shm\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/6bdb6f088c31b07fef1823acb79ef3de9dd935bb82cc1c34e4eeb0a172df229b/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546463391
Jul 17 03:28:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:55.249Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_net_dev_group\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"eno34np1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546459409
Jul 17 03:28:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:55.320Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_last_seen\", asset=\"msh-tsgx12\", asset_id=\"113\", container=\"coredns\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546460614
Jul 17 03:28:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:55.321Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_mtu_bytes\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"eno36np3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546463391
Jul 17 03:28:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:55.336Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_created\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"tsg-traffic-engine-numa-0-firewall\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546462058
Jul 17 03:28:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:55.343Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_annotations\", asset=\"msh-tsgx08\", asset_id=\"65\", configmap=\"prometheus-alertmanager\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546468950
Jul 17 03:28:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:55.350Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_mtu_bytes\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth4db46742\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546463391
Jul 17 03:28:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:55.368Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db0\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546470108
Jul 17 03:28:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:55.375Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_fifo_total\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"eno34np1\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546468328
Jul 17 03:28:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:55.412Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_writes_bytes_total\", asset=\"msh-tsgx08\", asset_id=\"65\", container=\"prometheus-pushgateway\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546456906
Jul 17 03:28:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:55.440Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_swap\", asset=\"msh-tsgx08\", asset_id=\"65\", container=\"kube-state-metrics\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546454480
Jul 17 03:28:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:55.471Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_device_id\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens2f0np0\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546468328
Jul 17 03:28:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:55.610Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_packets_dropped_total\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"cadvisor\", interface=\"ens4f1np1\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546459105
Jul 17 03:28:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:55.866Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_daemonset_status_current_number_scheduled\", asset=\"msh-tsgx12\", asset_id=\"113\", daemonset=\"dign-server\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546468663
Jul 17 03:28:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:55.892Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_created\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-1-firewall-6b6bf4679c\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546468663
Jul 17 03:28:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:55.915Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_packets_total\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth321ff728\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546463391
Jul 17 03:28:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:55.917Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx12\", asset_id=\"113\", container=\"dign-client\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Async\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546459630
Jul 17 03:28:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:55.938Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db8\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546458659
Jul 17 03:28:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:56.100Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_file_descriptors\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689546465397
Jul 17 03:28:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:56.145Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_compressed_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"vethc142f50b\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546459409
Jul 17 03:28:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:56.503Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"14\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546465883
Jul 17 03:28:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:56.820Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_resident_memory_bytes\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"tsg-clixon-restconf.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689546463372
Jul 17 03:28:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:57.233Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_mod_info\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"github.com/prometheus/procfs\", nz_agent_id=\"3\", program=\"systemd_exporter\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", version=\"v0.0.11\"}" timestamp=1689546463372
Jul 17 03:28:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:57.265Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Failed\", pod=\"helm-install-systemd-exporter-pw27n\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"6e4c6271-0705-4680-9a24-73bceb8107af\"}" timestamp=1689546473389
Jul 17 03:28:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:57.292Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_owner\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", owner_is_controller=\"true\", owner_kind=\"Deployment\", owner_name=\"tsg-traffic-engine-numa-1-firewall\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-1-firewall-7b57fdc5b7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546473389
Jul 17 03:28:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:57.298Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_working_set_bytes\", asset=\"pcap-tsgx03\", asset_id=\"73\", container=\"firewall\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689546462017
Jul 17 03:28:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:57.307Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"72\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546465883
Jul 17 03:28:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:57.611Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_free_bytes\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"shm\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/027350f54bb89061a60d10b405de81d85cf1dac7e54e96d6ac48c11ceaf660cd/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546465883
Jul 17 03:28:58 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:58.012Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"multi-user.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"target\"}" timestamp=1689546463372
Jul 17 03:28:58 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:58.269Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"overlay\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-d50a87f26946e9f4abd93f805eab86f9fd66b0a6be641385c5ecce42d8632b63-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689546463372
Jul 17 03:28:58 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:58.271Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_daemonset_status_number_available\", asset=\"pcap-tsgx03\", asset_id=\"73\", daemonset=\"dign-server\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546473389
Jul 17 03:28:58 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:58.365Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_hwmon_temp_celsius\", asset=\"pcap-tsgx03\", asset_id=\"73\", chip=\"pci0000:00_0000:00:19_3\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", sensor=\"temp10\", service=\"prometheus-node-exporter\"}" timestamp=1689546465883
Jul 17 03:28:58 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:58.653Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_hwmon_temp_celsius\", asset=\"pcap-tsgx03\", asset_id=\"73\", chip=\"pci0000:00_0000:00:18_3\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", sensor=\"temp8\", service=\"prometheus-node-exporter\"}" timestamp=1689546465883
Jul 17 03:28:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:59.238Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_scaling_frequency_min_hertz\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", cpu=\"10\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"OLAP-Node-Exporter-pcap-dellr740-dt001\", endpoint_id=\"251\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546477557
Jul 17 03:28:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:59.263Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_edac_csrow_uncorrectable_errors_total\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", controller=\"2\", csrow=\"unknown\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"OLAP-Node-Exporter-pcap-dellr740-dt001\", endpoint_id=\"251\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546477557
Jul 17 03:28:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:28:59.977Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"90\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546469083
Jul 17 03:29:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:00.034Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_gc_sys_bytes\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"OLAP-Node-Exporter-pcap-dellr740-dt001\", endpoint_id=\"251\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546477557
Jul 17 03:29:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:00.058Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"73\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546469083
Jul 17 03:29:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:00.078Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db219\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546465071
Jul 17 03:29:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:00.099Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_packets_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth2c4ec81a\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546469083
Jul 17 03:29:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:00.297Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_init_container_status_running\", asset=\"msh-tsgx01\", asset_id=\"51\", container=\"tsg-dign-init\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"dign-server-t2gqw\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"c6ffcf61-3641-43ac-b76c-b24b78ac0cd3\"}" timestamp=1689546467425
Jul 17 03:29:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:00.503Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2dpartuuid-42cfc368\\\\x2de341\\\\x2d4dfd\\\\x2da9a9\\\\x2d4f8343c4ae67.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689546464879
Jul 17 03:29:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:00.522Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_packets_dropped_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"cadvisor\", interface=\"eno2\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546472431
Jul 17 03:29:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:00.536Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_errors_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"cadvisor\", interface=\"virtio_dign_s\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546472431
Jul 17 03:29:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:00.577Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"1\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546469083
Jul 17 03:29:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:00.601Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_timer_last_trigger_seconds\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"mlocate-updatedb.timer\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689546464879
Jul 17 03:29:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:00.679Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Pending\", pod=\"helm-install-tsg-traffic-engine-numa-1-jg62v\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"9fb763c6-2ea9-4064-be90-c60d57c27d4f\"}" timestamp=1689546467425
Jul 17 03:29:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:00.760Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_bytes_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"cnibr0\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546469083
Jul 17 03:29:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:00.776Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_queue_length\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens8f0np0\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546469083
Jul 17 03:29:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:00.793Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"process_max_fds\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546469083
Jul 17 03:29:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:01.197Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens8f0np0\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546469083
Jul 17 03:29:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:02.644Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx02\", asset_id=\"63\", container=\"redis\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Read\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546456626
Jul 17 03:29:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:02.697Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"msh-tsgx02\", asset_id=\"63\", container=\"kube-multus\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"sleeping\"}" timestamp=1689546455006
Jul 17 03:29:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:02.728Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_errors_total\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", interface=\"eno2\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546470655
Jul 17 03:29:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:02.837Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx02\", asset_id=\"63\", cpu=\"87\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546478318
Jul 17 03:29:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:02.869Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_memory_HugePages_Rsvd\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546478318
Jul 17 03:29:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:02.878Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_net_dev_group\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"br_dign_c\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546478318
Jul 17 03:29:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:03.279Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx02\", asset_id=\"63\", cpu=\"66\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546478318
Jul 17 03:29:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:03.385Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_created\", asset=\"msh-tsgx02\", asset_id=\"63\", configmap=\"sce-tsg-traffic-engine-numa-1\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546471412
Jul 17 03:29:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:03.411Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_labels\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"chart-values-cm-policy-local-cache\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546471412
Jul 17 03:29:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:03.428Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_info\", asset=\"msh-tsgx02\", asset_id=\"63\", cluster_ip=\"10.43.252.29\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\", uid=\"bbf13329-fa7f-41a0-aadf-c7b26b39c34e\"}" timestamp=1689546471412
Jul 17 03:29:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:03.628Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_created\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"kubernetes-dashboard-key-holder\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546471412
Jul 17 03:29:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:03.653Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx02\", asset_id=\"63\", cpu=\"81\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546478318
Jul 17 03:29:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:03.676Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_files_free\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"shm\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/6eb0a7124e4e369f50d9c355b1faaad1b19d28e4aeb615a5b21734fa6c693cad/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546478318
Jul 17 03:29:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:03.690Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"ipmi_up\", asset=\"msh-tsgx02\", asset_id=\"63\", collector=\"ipmi\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"ipmi-exporter\"}" timestamp=1689546474732
Jul 17 03:29:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:03.754Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"bucket_objects_histogram\", asset=\"pcap-kvm-cm001\", asset_id=\"45\", bucket=\"auditlog\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"CM-Minio-pcap-kvm-cm001\", endpoint_id=\"301\", module=\"CM-Minio\", module_id=\"63\", nz_agent_id=\"3\", object_size=\"BETWEEN_64_MB_AND_128_MB\", parent_asset=\"pcap-dellr740-pve001\", parent_asset_id=\"43\", project=\"Central-Management\", project_id=\"5\"}" timestamp=1689546482056
Jul 17 03:29:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:03.932Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Succeeded\", pod=\"svclb-traefik-1a2acd8b-r8v4z\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"80a49efb-cd3a-4d5f-8345-2cc7583d6815\"}" timestamp=1689546471412
Jul 17 03:29:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:04.181Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_netstat_Udp6_InDatagrams\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546478318
Jul 17 03:29:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:04.202Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db255\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546469269
Jul 17 03:29:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:04.312Z caller=dedupe.go:112 component=remote level=info remote_name=52db6f url=http://10.10.10.159:10090/nz-agent/prometheus/proxy/api/v1/write msg="Dropped sample for series that was not explicitly dropped via relabelling" ref=4436650
Jul 17 03:29:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:06.643Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cpu=\"36\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546483910
Jul 17 03:29:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:06.646Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_threads\", asset=\"twa-tsgx01\", asset_id=\"91\", container=\"configmap-reloader\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546474405
Jul 17 03:29:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:06.685Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cpu=\"79\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546483910
Jul 17 03:29:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:06.824Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_packets_dropped_total\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"cadvisor\", interface=\"ens8f1np1\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546482685
Jul 17 03:29:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:07.083Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_created\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"dign-server-znxnp\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"7b4d8362-f626-49d3-b132-7bb684fd42a1\"}" timestamp=1689546481227
Jul 17 03:29:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:07.391Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cpu=\"26\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546483910
Jul 17 03:29:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:07.439Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cpu=\"117\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546483910
Jul 17 03:29:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:07.477Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-infiniband-umad5.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689546473582
Jul 17 03:29:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:07.486Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cpu=\"64\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546483910
Jul 17 03:29:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:07.505Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_max_fds\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dbus.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689546473582
Jul 17 03:29:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:07.511Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_allocator_rss_ratio\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546478290
Jul 17 03:29:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:07.519Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-7ce93833\\\\x2d4c85\\\\x2d4a59\\\\x2dbe25\\\\x2d1ba1b6f91e91-volumes-kubernetes.io\\\\x7esecret-cm\\\\x2dcache\\\\x2dextra\\\\x2dsecrets.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689546473582
Jul 17 03:29:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:07.525Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db42\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546478290
Jul 17 03:29:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:07.608Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_bytes_total\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"cadvisor\", interface=\"ens1f0np0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546482685
Jul 17 03:29:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:07.630Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_bytes_total\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"cadvisor\", interface=\"eno34np1\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546482685
Jul 17 03:29:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:07.641Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link_mode\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"eno2\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546483910
Jul 17 03:29:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:07.700Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Line_Pkt\", app_name=\"sapp4\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", vsysId=\"1\"}" timestamp=1689546484123
Jul 17 03:29:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:07.715Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"home.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689546473582
Jul 17 03:29:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:07.931Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_annotations\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"chart-values-systemd-exporter\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546481227
Jul 17 03:29:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:07.954Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-virtual-net-vethdd29a3ce.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689546473582
Jul 17 03:29:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:08.370Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"clickhouse_network_send_packets_vethae692de\", asset=\"msh-dellr740-olap002\", asset_id=\"13\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Clickhouse-msh-dellr740-olap002\", endpoint_id=\"73\", module=\"NC-Clickhouse\", module_id=\"29\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546487759
Jul 17 03:29:09 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:09.934Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_CompactionOutputFileCount_99_9th_percentile\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Server\"}" timestamp=1689546488804
Jul 17 03:29:09 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:09.963Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_maxFlushQueueSize\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"knowledge_base_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"4e53ab9d4bf390c647df30577df37a64\", table=\"index_time_knowledge_base_hos_bucket\", type=\"index\"}" timestamp=1689546488804
Jul 17 03:29:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:10.015Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_deleteCount\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"firewall_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"8a4c8c9df530241b5d7eaf5c4ed6ed9b\", table=\"index_time_firewall_hos_bucket\", type=\"index\"}" timestamp=1689546488804
Jul 17 03:29:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:10.089Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_maxCompactionQueueSize\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"session_record_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"8624285d7679e25fa47769ce6fc2c350\", table=\"index_filename_session_record_hos_bucket\", type=\"index\"}" timestamp=1689546488804
Jul 17 03:29:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:10.143Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"java_lang_GarbageCollector_LastGcInfo_memoryUsageAfterGc_max\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", key=\"CMS Old Gen\", module=\"DC-HBase\", module_id=\"3\", name=\"ParNew\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546488804
Jul 17 03:29:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:11.122Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_readRequestCount\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"rtp_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"bb21b090347b6a8fc3b90ef139c41f67\", table=\"index_filename_rtp_hos_bucket\", type=\"index\"}" timestamp=1689546488804
Jul 17 03:29:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:11.312Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_numReferenceFiles\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"firewall_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"7825e373ea56537e53b94051524e36b0\", table=\"index_filename_firewall_hos_bucket\", type=\"index\"}" timestamp=1689546490148
Jul 17 03:29:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:11.314Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_MUTEX_metric_getTime_mean\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546490148
Jul 17 03:29:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:11.327Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_numReferenceFiles\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"assessment_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"db3245bccdc6403a0161f317bbd97ed6\", table=\"index_filename_assessment_hos_bucket\", type=\"index\"}" timestamp=1689546490148
Jul 17 03:29:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:11.338Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_3_metric_storeFileSize\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546490148
Jul 17 03:29:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:11.350Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_default_metric_splitTime_num_ops\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546490148
Jul 17 03:29:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:11.363Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_readRequestCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"a3a11b5a21a8f6f027bd4558fdd16808\", table=\"proxy_hos_bucket\", type=\"data\"}" timestamp=1689546490148
Jul 17 03:29:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:11.375Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_4_metric_majorCompactionTime_num_ops\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546490148
Jul 17 03:29:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:11.401Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_default_metric_majorCompactionInputSize_90th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546490148
Jul 17 03:29:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:11.416Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_PutBatch_min\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Server\"}" timestamp=1689546490148
Jul 17 03:29:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:11.507Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_session_record_hos_bucket_metric_flushedMemstoreBytes\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546490148
Jul 17 03:29:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:11.640Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_scanCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"4d6e638e9e2116f7f9ad175dc85b32f3\", table=\"session_record_hos_bucket\", type=\"data\"}" timestamp=1689546490148
Jul 17 03:29:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:11.664Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_storeCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"knowledge_base_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"408298ac4baa4d15f68738efc36b7451\", table=\"index_filename_knowledge_base_hos_bucket\", type=\"index\"}" timestamp=1689546490148
Jul 17 03:29:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:11.695Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_2_metric_scanTime_25th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546490148
Jul 17 03:29:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:11.699Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"pcap-tsgx08\", asset_id=\"87\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx08\", endpoint_id=\"481\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"1\", phase=\"Failed\", pod=\"prometheus-alertmanager-6df7f457b6-9nzwq\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"30dec72e-d642-42f9-8896-7f2bc14ef7c5\"}" timestamp=1689546477534
Jul 17 03:29:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:11.751Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_session_record_hos_bucket_metric_majorCompactionOutputSize_90th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546490148
Jul 17 03:29:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:11.780Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_193_1_metric_scanSize_99th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546490148
Jul 17 03:29:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:11.794Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_numBytesCompactedCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"knowledge_base_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"a5ee01623c98149e761abaf78e7b7549\", table=\"index_time_knowledge_base_hos_bucket\", type=\"index\"}" timestamp=1689546490148
Jul 17 03:29:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:11.889Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_tsg_galaxy_table_relation_framedip_account_metric_incrementTime_99_9th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546490148
Jul 17 03:29:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:11.971Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_appendCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"96a55ea6420b424b55926f18a013afc6\", table=\"firewall_hos_bucket\", type=\"data\"}" timestamp=1689546490148
Jul 17 03:29:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:12.005Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_4_metric_getTime_max\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546490148
Jul 17 03:29:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:12.284Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_session_record_hos_bucket_metric_maxStoreFileAge\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546490148
Jul 17 03:29:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:12.425Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_session_record_hos_bucket_metric_getTime_75th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546490148
Jul 17 03:29:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:12.499Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_BlockedFlushes_min\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Memory\"}" timestamp=1689546490148
Jul 17 03:29:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:12.527Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_1_metric_flushTime_95th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546490148
Jul 17 03:29:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:12.597Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_194_2_metric_majorCompactionTime_25th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546490148
Jul 17 03:29:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:12.656Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_default_metric_compactionOutputFileCount_mean\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546490148
Jul 17 03:29:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:15.356Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-eno33np0.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689546487046
Jul 17 03:29:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:15.968Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-e026c1a65633a8a9f1f06fe4c43008960cbf64ced235e8292b3b355c4b39b3e2.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"scope\"}" timestamp=1689546487046
Jul 17 03:29:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:16.247Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_annotations\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-1-firewall-774cf75f79\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546488076
Jul 17 03:29:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:16.293Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db212\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546486027
Jul 17 03:29:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:16.699Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_changes_total\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens4f0np0\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546481230
Jul 17 03:29:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:16.719Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db52\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546486027
Jul 17 03:29:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:16.780Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-hwdb-update.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689546487046
Jul 17 03:29:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:16.782Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"scrape_series_added\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_instance=\"10.42.0.62:9290\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"ipmi-exporter\"}" timestamp=1689546484471
Jul 17 03:29:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:16.787Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-fsck-root.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"service\"}" timestamp=1689546487046
Jul 17 03:29:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:16.838Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_timex_pps_stability_exceeded_total\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546481230
Jul 17 03:29:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:16.959Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_size_bytes\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"shm\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/554896d488f4e8a2f24d4292c4f8e7737fee9bfa4a62e85997c3eeb92497fabb/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546481230
Jul 17 03:29:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:16.975Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_multicast_total\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethe34782c1\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546481230
Jul 17 03:29:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:20.029Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_up\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens6f1np1\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546497612
Jul 17 03:29:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:20.109Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"85\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546497612
Jul 17 03:29:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:20.153Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_protocol_type\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth9d00126e\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546497612
Jul 17 03:29:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:20.292Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-sda1.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689546484356
Jul 17 03:29:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:20.310Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"nfsdcld.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689546484356
Jul 17 03:29:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:20.364Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_multicast_packets_received_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"mlx5_0\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546488263
Jul 17 03:29:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:20.376Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"phy_rx_error_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", dev=\"ens7f0np0\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"mrzcpd-exporter\"}" timestamp=1689546494036
Jul 17 03:29:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:20.513Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"121\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546497612
Jul 17 03:29:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:20.554Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_size_bytes\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"shm\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/460f5be153f52052cd29535bc0132c4b771599ff123f1c6a334c8908afcd72a4/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546497612
Jul 17 03:29:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:20.574Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_tasks_current\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"udisks2.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689546484356
Jul 17 03:29:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:20.602Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_created\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_endpoint=\"kubernetes\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546493949
Jul 17 03:29:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:20.622Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_compressed_total\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"cnibr0\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546497612
Jul 17 03:29:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:20.635Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_virtual_memory_bytes\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"tsg-clixon-backend.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689546484356
Jul 17 03:29:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:20.637Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"15\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546497612
Jul 17 03:29:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:20.651Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_carrier_total\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"virtio_dign_c\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546497612
Jul 17 03:29:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:20.654Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"pcap-tsgx10\", asset_id=\"89\", condition=\"Available\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"tsg-traffic-engine-numa-1-firewall\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"unknown\"}" timestamp=1689546493828
Jul 17 03:29:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:20.664Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_sockstat_TCP_tw\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546497612
Jul 17 03:29:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:20.724Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_init_container_status_ready\", asset=\"pcap-tsgx10\", asset_id=\"89\", container=\"firewall-init\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", pod=\"tsg-traffic-engine-numa-1-firewall-5d7bfbbbfb-4wgxr\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"ab53e5a7-f22e-498d-b5af-535ae7359350\"}" timestamp=1689546493828
Jul 17 03:29:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:20.788Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-traefik-b7tft\", project=\"TSG-OS\", project_id=\"7\", reason=\"NodeAffinity\", service=\"prometheus-kube-state-metrics\", uid=\"f8334eab-55b4-4187-9700-4ad2897e2d91\"}" timestamp=1689546493828
Jul 17 03:29:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:20.897Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"scrape_samples_scraped\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_instance=\"10.42.0.97:9290\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"ipmi-exporter\"}" timestamp=1689546488448
Jul 17 03:29:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:20.909Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_cpu_seconds_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"slice\"}" timestamp=1689546490657
Jul 17 03:29:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:20.919Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-eno1.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689546490657
Jul 17 03:29:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:20.923Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-80be009ae9aab0270cea9b98d94e0627ae6f5e2d2d66570b5bba92ebaf1ceed2.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"scope\"}" timestamp=1689546490657
Jul 17 03:29:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:20.929Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"ext4\", name=\"data-var-lib-kubelet-pods-e12ed51a\\\\x2d28b1\\\\x2d413a\\\\x2d82be\\\\x2d8a0fcac5cffa-volume\\\\x2dsubpaths-config\\\\x2dpath-dign\\\\x2dserver\\\\x2ddns-1.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689546490657
Jul 17 03:29:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:21.820Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"msh-tsgx07\", asset_id=\"53\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx07\", endpoint_id=\"421\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"1\", phase=\"Running\", pod=\"local-path-provisioner-957fdf8bc-8kpm2\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"952911e2-03f2-4ca8-a299-b802489102c5\"}" timestamp=1689546491835
Jul 17 03:29:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:21.831Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"msh-tsgx07\", asset_id=\"53\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx07\", endpoint_id=\"421\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"ext4\", name=\"data-var-lib-kubelet-pods-db00ee1f\\\\x2d9999\\\\x2d4d76\\\\x2db400\\\\x2db34f14313057-volume\\\\x2dsubpaths-config\\\\x2dvolume-telegraf-2.mount\", nz_agent_id=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689546491532
Jul 17 03:29:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:23.352Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"java_lang_MemoryPool_Valid\", asset=\"msh-dellr740-dt003\", asset_id=\"9\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-Yarn-msh-dellr740-dt003-NodeManager\", endpoint_id=\"355\", module=\"DC-Yarn\", module_id=\"1\", name=\"Metaspace\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546502405
Jul 17 03:29:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:23.469Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_device_id\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"cnibr0\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546494790
Jul 17 03:29:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:23.499Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Send_Tcp_Rst\", app_name=\"sapp4\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", vsysId=\"1\"}" timestamp=1689546498016
Jul 17 03:29:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:23.906Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Ipv6_Pkt\", app_name=\"sapp4\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", vsysId=\"1\"}" timestamp=1689546498016
Jul 17 03:29:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:24.002Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_NodeManager_MemHeapUsedM\", asset=\"msh-dellr740-dt003\", asset_id=\"9\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-Yarn-msh-dellr740-dt003-NodeManager\", endpoint_id=\"355\", module=\"DC-Yarn\", module_id=\"1\", name=\"JvmMetrics\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546502405
Jul 17 03:29:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:24.102Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"53\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546500160
Jul 17 03:29:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:24.137Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_compressed_total\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth0c6b6d72\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546494790
Jul 17 03:29:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:24.143Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-a27a1ebc\\\\x2d016c\\\\x2d4998\\\\x2da3f3\\\\x2d85e865dfcd16-volume\\\\x2dsubpaths-telegraf-telegraf-0.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689546491102
Jul 17 03:29:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:24.152Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_queue_length\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"vethd027ff41\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546494790
Jul 17 03:29:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:24.181Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"ftx_missed_total\", app=\"sapp4-1\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"nf_1_fw\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"mrzcpd-exporter\", serviceFunctionIndex=\"1\"}" timestamp=1689546495378
Jul 17 03:29:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:24.197Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_owner\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", owner_is_controller=\"true\", owner_kind=\"ReplicaSet\", owner_name=\"configmap-reloader-6c55bbc49f\", pod=\"configmap-reloader-6c55bbc49f-fh258\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"f7af3cd7-b972-4aba-a12a-7b5c827e57eb\"}" timestamp=1689546491122
Jul 17 03:29:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:24.199Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_oom_events_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"lb-tcp-80\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546478776
Jul 17 03:29:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:24.368Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_mapped_file\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"dign-server-web\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546489419
Jul 17 03:29:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:24.387Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", state=\"sleeping\"}" timestamp=1689546489456
Jul 17 03:29:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:24.413Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_complete\", asset=\"msh-tsgx09\", asset_id=\"55\", condition=\"true\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", job_name=\"helm-install-systemd-exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546500068
Jul 17 03:29:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:24.429Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_last_seen\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"telegraf-security\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689546503435
Jul 17 03:29:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:24.435Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_errs_total\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens3f1np1\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546494790
Jul 17 03:29:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:24.484Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"37\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546500160
Jul 17 03:29:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:24.497Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"msh-tsgx09\", asset_id=\"55\", container=\"configmap-reloader\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"iowaiting\"}" timestamp=1689546491884
Jul 17 03:29:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:24.547Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_fifo_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"vethc31b7dd7\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546500160
Jul 17 03:29:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:24.563Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_mtu_bytes\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"eno33np0\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546494790
Jul 17 03:29:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:24.601Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_scrape_collector_success\", asset=\"msh-tsgx09\", asset_id=\"55\", collector=\"stat\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546494790
Jul 17 03:29:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:24.607Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546500160
Jul 17 03:29:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:24.654Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"tsg_master_T_fail_log\", app_name=\"tsg_master\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"traffic-engine\", line_name=\"INTERNAL-RTP-RECORD\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", table_name=\"sendlog\", vsysId=\"1\"}" timestamp=1689546500446
Jul 17 03:29:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:24.669Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-user-sessions.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"service\"}" timestamp=1689546498416
Jul 17 03:29:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:24.712Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"6\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546500160
Jul 17 03:29:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:24.723Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_flags\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"vethc96a0a69\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546500160
Jul 17 03:29:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:24.738Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"configmap-reloader-6c55bbc49f-fh258\", project=\"TSG-OS\", project_id=\"7\", reason=\"Evicted\", service=\"prometheus-kube-state-metrics\", uid=\"f7af3cd7-b972-4aba-a12a-7b5c827e57eb\"}" timestamp=1689546491122
Jul 17 03:29:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:24.812Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"16\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546500160
Jul 17 03:29:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:24.829Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_io_threaded_writes_processed\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546501795
Jul 17 03:29:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:24.942Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_transmit_wait_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"mlx5_5\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546500160
Jul 17 03:29:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:24.959Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_up_changes_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"cnibr0\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546500160
Jul 17 03:29:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:25.143Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_protocol_type\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"eno33np0\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546500160
Jul 17 03:29:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:25.169Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db123\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546501795
Jul 17 03:29:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:25.234Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_packets_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"eno33np0\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546500160
Jul 17 03:29:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:25.407Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"118\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546500160
Jul 17 03:29:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:26.277Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_daemonset_annotations\", asset=\"twa-tsgx09\", asset_id=\"107\", daemonset=\"svclb-traefik-23e75809\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546498744
Jul 17 03:29:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:26.535Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_annotations\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.cm-policy-local-cache.v1\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546498744
Jul 17 03:29:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:26.677Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"cm-policy-local-cache-redis-master\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546498744
Jul 17 03:29:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:27.026Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Tcp_Bit\", app_name=\"sapp4\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", vsysId=\"1\"}" timestamp=1689546495421
Jul 17 03:29:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:27.036Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_constraint_errors_transmitted_total\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_11\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546495810
Jul 17 03:29:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:27.157Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_created\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-cm-policy-local-cache-wzw8p\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"57683893-10c5-486b-89a6-dbc1429ed965\"}" timestamp=1689546498744
Jul 17 03:29:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:27.172Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled_time\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-systemd-exporter-xvbr4\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"3f2ca409-551c-41d4-bd2d-4298cbf936c3\"}" timestamp=1689546498744
Jul 17 03:29:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:27.175Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_labels\", asset=\"twa-tsgx09\", asset_id=\"107\", configmap=\"shaping-tsg-traffic-engine-numa-1\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546498744
Jul 17 03:29:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:27.294Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_restart_policy\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-alertmanager-6df7f457b6-77pk9\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", type=\"Always\", uid=\"2edb1c6e-4f58-4c31-b8c5-28dfc8f91f4d\"}" timestamp=1689546498744
Jul 17 03:29:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:27.303Z caller=dedupe.go:112 component=remote level=error remote_name=52db6f url=http://10.10.10.159:10090/nz-agent/prometheus/proxy/api/v1/write msg="non-recoverable error" count=1000 exemplarCount=0 err="server returned HTTP status 400 : out of order sample"
Jul 17 03:29:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:27.310Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"abrt-xorg.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"simple\", type=\"service\"}" timestamp=1689546493907
Jul 17 03:29:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:27.395Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_labels\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-systemd-exporter-xvbr4\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"3f2ca409-551c-41d4-bd2d-4298cbf936c3\"}" timestamp=1689546498744
Jul 17 03:29:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:27.486Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_fifo_total\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens3f0np0\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546495810
Jul 17 03:29:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:27.489Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-848571bc\\\\x2da31e\\\\x2d44de\\\\x2da0f0\\\\x2dd172cf8f3bd0-volume\\\\x2dsubpaths-config\\\\x2dvolume-firewall-1.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689546494114
Jul 17 03:29:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:27.576Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Pending\", pod=\"svclb-traefik-23e75809-pm7r4\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"b2322992-daa4-43ba-897c-0532360cf5ea\"}" timestamp=1689546498744
Jul 17 03:29:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:27.624Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_last_seen\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"prometheus-alertmanager-configmap-reload\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546504864
Jul 17 03:29:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:27.830Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_created\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"kube-multus-ds-ntb9l\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"396cb6af-9b76-4a32-aad2-deb6dbe16f5b\"}" timestamp=1689546495682
Jul 17 03:29:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:27.854Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"117\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546502882
Jul 17 03:29:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:28.519Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_created\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", job_name=\"helm-install-traefik-crd\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546495682
Jul 17 03:29:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:28.536Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_reservation_limit_bytes\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"prometheus-pushgateway\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546504841
Jul 17 03:29:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:28.654Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-46b1016fe0882f640c1330cdff069a4aa4775b598d30e8571ff0c78026146d00-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689546494114
Jul 17 03:29:29 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:29.044Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"58\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546502882
Jul 17 03:29:30 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:30.944Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"80\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546500939
Jul 17 03:29:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:31.497Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db168\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546510141
Jul 17 03:29:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:31.585Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"86\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546499093
Jul 17 03:29:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:31.597Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_errs_total\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens2f0np0\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546499093
Jul 17 03:29:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:31.661Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_last_seen\", asset=\"twa-tsgx03\", asset_id=\"95\", container=\"dashboard-metrics-scraper\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546503150
Jul 17 03:29:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:31.665Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_init_container_status_waiting\", asset=\"msh-tsgx05\", asset_id=\"67\", container=\"firewall-init\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", pod=\"tsg-traffic-engine-numa-0-firewall-76b5c5788d-2cs6q\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"99e7b7f0-307f-4274-a670-faf429f178f7\"}" timestamp=1689546504068
Jul 17 03:29:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:31.684Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"22\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546499093
Jul 17 03:29:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:31.719Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx03\", asset_id=\"95\", container=\"kubernetes-dashboard\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Async\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546487116
Jul 17 03:29:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:31.739Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_usage_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"total\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546502864
Jul 17 03:29:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:31.777Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"0\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546499093
Jul 17 03:29:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:31.842Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_sockets\", asset=\"twa-tsgx03\", asset_id=\"95\", container=\"metrics-server\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546493830
Jul 17 03:29:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:31.851Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"11\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546500939
Jul 17 03:29:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:31.883Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_restarts_total\", asset=\"twa-tsgx03\", asset_id=\"95\", container=\"metrics\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"cm-policy-local-cache-redis-master-69967b5879-x29xl\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"1c67a2da-71e5-4d96-889e-b1eefe2ff7af\"}" timestamp=1689546510436
Jul 17 03:29:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:31.911Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"initrd-fs.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"target\"}" timestamp=1689546506907
Jul 17 03:29:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:31.917Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"svclb-traefik-acf36120-2ckwj\", project=\"TSG-OS\", project_id=\"7\", reason=\"NodeAffinity\", service=\"prometheus-kube-state-metrics\", uid=\"14f66a9f-85bf-4104-ac3f-c15dda542ad5\"}" timestamp=1689546510436
Jul 17 03:29:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:31.967Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"123\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546499093
Jul 17 03:29:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:32.018Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_created\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.tsg-traffic-engine-numa-1.v4\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546510436
Jul 17 03:29:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:32.026Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db107\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546506438
Jul 17 03:29:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:32.045Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_threads_max\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"prometheus-pushgateway\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546489654
Jul 17 03:29:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:32.049Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546499093
Jul 17 03:29:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:32.082Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"nfs-server.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"oneshot\", type=\"service\"}" timestamp=1689546506907
Jul 17 03:29:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:32.151Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"62\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546499093
Jul 17 03:29:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:32.234Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_terminated\", asset=\"msh-tsgx05\", asset_id=\"67\", container=\"local-path-provisioner\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"local-path-provisioner-957fdf8bc-hv6cn\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"73c566ef-6d83-4dd3-9f97-e8eda48fa60f\"}" timestamp=1689546504068
Jul 17 03:29:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:32.262Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Succeeded\", pod=\"helm-install-ipmi-exporter-n6tms\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"0855a18d-1839-4129-9de2-c38efca608a7\"}" timestamp=1689546504068
Jul 17 03:29:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:32.319Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-ttyS1.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689546501015
Jul 17 03:29:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:32.389Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"126\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546500939
Jul 17 03:29:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:32.425Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_drop_total\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"lo\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546500939
Jul 17 03:29:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:32.460Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"46\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546500939
Jul 17 03:29:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:32.484Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_multicast_packets_received_total\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_4\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546499093
Jul 17 03:29:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:32.489Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_errs_total\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"bond0\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546500939
Jul 17 03:29:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:32.524Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"116\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546499093
Jul 17 03:29:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:32.526Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_bytes_total\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth09efe48d\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546499093
Jul 17 03:29:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:32.527Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_spec_replicas\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"local-path-provisioner-957fdf8bc\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546499894
Jul 17 03:29:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:32.573Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-0563c65c7fb8336d933d96e976bd440587e91e8b942ae41fc5e8ad16c8376f5f.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"scope\"}" timestamp=1689546506907
Jul 17 03:29:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:32.597Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_write_seconds_total\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda2\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546495279
Jul 17 03:29:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:32.650Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_created\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-headless\", uid=\"6ec3df27-f9ee-487c-bd3a-256bf48c74d9\"}" timestamp=1689546499276
Jul 17 03:29:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:32.666Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_colls_total\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens1f1np1\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546500605
Jul 17 03:29:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:32.676Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_dormant\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens4f2np2\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546500791
Jul 17 03:29:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:32.805Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_commands_rejected_calls_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cmd=\"del\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546509260
Jul 17 03:29:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:32.955Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"30\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546500605
Jul 17 03:29:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:32.962Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_metadata_resource_version\", asset=\"twa-tsgx03\", asset_id=\"95\", configmap=\"local-path-config\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546510436
Jul 17 03:29:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:32.968Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:e0-0000:e0:05.1-0000:e1:00.0-net-eno1.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689546506907
Jul 17 03:29:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:32.975Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_waiting\", asset=\"twa-tsgx03\", asset_id=\"95\", container=\"helm\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-ipmi-exporter-h7mdw\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"14488225-fd95-4f38-a9af-d4af570ca0f1\"}" timestamp=1689546510436
Jul 17 03:29:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:32.978Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_rate_bytes_per_second\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_0\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546500791
Jul 17 03:29:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:32.983Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"scrape_duration_seconds\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_instance=\"localhost\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546503123
Jul 17 03:29:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:32.992Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-initctl.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"service\"}" timestamp=1689546506907
Jul 17 03:29:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:33.020Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_commands_duration_seconds_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cmd=\"rename\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546496711
Jul 17 03:29:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:33.027Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"twa-tsgx03\", asset_id=\"95\", condition=\"unknown\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"dign-client-nvg4d\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"6e410083-6316-4234-8b1e-7c79ee4588d4\"}" timestamp=1689546510436
Jul 17 03:29:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:33.039Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_name_assign_type\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"bond0\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546500791
Jul 17 03:29:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:33.052Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_compressed_total\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens4f2np2\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546500791
Jul 17 03:29:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:33.230Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_ulimits_soft\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"dign-client\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", ulimit=\"max_open_files\"}" timestamp=1689546485937
Jul 17 03:29:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:33.267Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_type\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.traefik-crd.v1\", service=\"prometheus-kube-state-metrics\", type=\"helm.sh/release.v1\"}" timestamp=1689546499276
Jul 17 03:29:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:33.290Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_cpu_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"snmpd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689546507691
Jul 17 03:29:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:33.314Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.grpc.v1.cri-sandboxes-81835378c7e69e62b6d51929d9f019f537c50bb1d541bdddbc412a16a44adc90-shm.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689546506907
Jul 17 03:29:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:33.349Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"timers.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"target\"}" timestamp=1689546506907
Jul 17 03:29:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:33.411Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_gc_duration_seconds_sum\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546509260
Jul 17 03:29:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:33.425Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_labels\", asset=\"twa-tsgx05\", asset_id=\"99\", configmap=\"kube-root-ca.crt\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-public\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546499894
Jul 17 03:29:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:33.443Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_start_time\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-alertmanager-6df7f457b6-49ggg\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"32b13371-dbfd-40eb-bd71-e35dd195f135\"}" timestamp=1689546499894
Jul 17 03:29:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:33.524Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_goroutines\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"ipmi-exporter\"}" timestamp=1689546503498
Jul 17 03:29:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:33.543Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_mcache_inuse_bytes\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546500791
Jul 17 03:29:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:33.568Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"fcoe.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689546507691
Jul 17 03:29:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:33.583Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_terminated\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"helm\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-configmap-reloader-nzdfr\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"e6f96c8f-7d00-4688-84c4-916e97112e6b\"}" timestamp=1689546499276
Jul 17 03:29:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:33.704Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_readonly\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"shm\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/a98f3e8875a4e0eb7a00a1677df4b100c59d90548cb5d22905911c3bc6896c33/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546500791
Jul 17 03:29:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:33.798Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_usage_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", container=\"lb-tcp-80\", cpu=\"total\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546492063
Jul 17 03:29:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:33.820Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"79\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546500605
Jul 17 03:29:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:33.831Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_changes_total\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens1f0np0\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546500605
Jul 17 03:29:36 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:36.395Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"java_lang_GarbageCollector_LastGcInfo_memoryUsageBeforeGc_max\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-Hmaster\", endpoint_id=\"347\", key=\"Par Eden Space\", module=\"DC-HBase\", module_id=\"3\", name=\"ParNew\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546514852
Jul 17 03:29:38 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:38.682Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"twa-dellr740-dt003\", asset_id=\"41\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"OLAP-Node-Exporter-twa-dellr740-dt003\", endpoint_id=\"261\", module=\"OLAP-Node-Exporter\", module_id=\"51\", name=\"6\", nz_agent_id=\"3\", olap_node_exporter=\"hbase_master,clickhouse_data,kafka,gohangout,zookeeper,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\", type=\"Processor\"}" timestamp=1689546518473
Jul 17 03:29:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:40.528Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_namespace_status_phase\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Terminating\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546520034
Jul 17 03:29:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:40.594Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_running\", asset=\"twa-tsgx04\", asset_id=\"97\", container=\"local-path-provisioner\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"local-path-provisioner-957fdf8bc-t9pfz\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"5b3f6272-57d2-4b0d-83be-046bdee2c23b\"}" timestamp=1689546520034
Jul 17 03:29:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:40.614Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-08d6039e72ec2780ba9e1efcfc1bd71aa77e230c35996dd86c86cb1272ce92e2.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"scope\"}" timestamp=1689546515764
Jul 17 03:29:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:40.890Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_10_193_1_metric_compactionOutputSize_90th_percentile\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546519758
Jul 17 03:29:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:41.035Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"phy_rx_error_total\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", dev=\"ens7f1np1\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"mrzcpd-exporter\"}" timestamp=1689546508913
Jul 17 03:29:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:41.433Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_working_set_bytes\", asset=\"twa-tsgx04\", asset_id=\"97\", container=\"prometheus-node-exporter\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546500282
Jul 17 03:29:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:41.442Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_changes_total\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens3f0np0\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546511647
Jul 17 03:29:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:41.457Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2did-wwn\\\\x2d0x62cea7f08346c5002c26f9f8d57b1b40.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689546515764
Jul 17 03:29:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:41.545Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_labels\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"prometheus-pushgateway-bd476bcf9\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546520034
Jul 17 03:29:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:41.548Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"52\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546511647
Jul 17 03:29:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:41.670Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"93\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546511647
Jul 17 03:29:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:41.842Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"62\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546511647
Jul 17 03:29:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:41.866Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-virtual-net-br_dign_c.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689546515764
Jul 17 03:29:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:42.034Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"paths.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"target\"}" timestamp=1689546515764
Jul 17 03:29:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:42.472Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db123\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546514838
Jul 17 03:29:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:42.488Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-5611dc73bcd87d48480acece3af195cc66b4bca02907f9bb4b42e627b59bd1c9.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"scope\"}" timestamp=1689546520671
Jul 17 03:29:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:42.572Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"java_lang_MemoryPool_Usage_max\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"CMS Old Gen\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546519758
Jul 17 03:29:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:42.639Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Delete_99th_percentile\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Server\"}" timestamp=1689546519758
Jul 17 03:29:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:43.103Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", cpu=\"35\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546516242
Jul 17 03:29:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:43.484Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_drop_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth585262fb\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546516242
Jul 17 03:29:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:43.574Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-pod27368c72_efff_405c_9fe0_1a20af809631.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"slice\"}" timestamp=1689546520671
Jul 17 03:29:44 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:44.047Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", cpu=\"24\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546516242
Jul 17 03:29:44 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:44.062Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_scrape_collector_duration_seconds\", asset=\"pcap-tsgx01\", asset_id=\"71\", collector=\"netdev\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546516242
Jul 17 03:29:48 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:48.839Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_tsg_galaxy_table_relation_account_framedip_metric_deleteBatchTime_99_9th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546527438
Jul 17 03:29:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:49.156Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_3_metric_compactionTime_99_9th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546527438
Jul 17 03:29:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:49.171Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_memStoreSize\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"proxy_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"5648ed44ac1d1626415660c3c36fbd73\", table=\"index_time_proxy_hos_bucket\", type=\"index\"}" timestamp=1689546527438
Jul 17 03:29:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:49.172Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_PostIndexUpdateTime_min\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"PhoenixIndexer\"}" timestamp=1689546527438
Jul 17 03:29:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:49.221Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_flushesQueuedCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"assessment_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"5d0defd99486c92aa30397e891e99c71\", table=\"index_partfile_assessment_hos_bucket\", type=\"index\"}" timestamp=1689546527438
Jul 17 03:29:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:49.234Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_compactionsCompletedCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"session_record_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"5e4a8e5af9d1dd712c1dd2749d210276\", table=\"index_partfile_session_record_hos_bucket\", type=\"index\"}" timestamp=1689546527438
Jul 17 03:29:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:49.500Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"tmpfs\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/var/lib/kubelet/pods/f930767b-550c-482d-9c10-4ec670104c56/volumes/kubernetes.io~projected/kube-api-access-888qm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546521797
Jul 17 03:29:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:49.528Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db141\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546526625
Jul 17 03:29:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:49.561Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_id\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth1dedddbd\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546521797
Jul 17 03:29:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:49.589Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"twa-tsgx10\", asset_id=\"109\", cpu=\"122\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546521797
Jul 17 03:29:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:49.684Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-107277e51994fc3b93285c23e3c794fe3985e14e0a360b66c3826c666f5d0b77.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"scope\"}" timestamp=1689546521057
Jul 17 03:29:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:49.751Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_194_4_metric_regionCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546527438
Jul 17 03:29:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:49.789Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx10\", asset_id=\"109\", cpu=\"18\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546521797
Jul 17 03:29:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:49.800Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2dlabel-TGS\\\\x2dOS\\\\x2dSYSROOT.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689546521057
Jul 17 03:29:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:49.838Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_compactionsQueuedCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"troubleshooting_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"5f2da531a09790e39db22d18390c8a5d\", table=\"index_filename_troubleshooting_hos_bucket\", type=\"index\"}" timestamp=1689546527438
Jul 17 03:29:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:49.946Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"tmpfs\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/var/lib/kubelet/pods/6f7082c9-24a1-4f4f-af11-ba474eb9a99b/volumes/kubernetes.io~projected/kube-api-access-m8hpv\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546526709
Jul 17 03:29:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:49.968Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_carrier_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens8f0np0\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546526709
Jul 17 03:29:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:49.980Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_bytes_total\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", interface=\"ens8f0np0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546517654
Jul 17 03:29:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:49.999Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_edac_csrow_uncorrectable_errors_total\", asset=\"twa-tsgx10\", asset_id=\"109\", controller=\"0\", csrow=\"unknown\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546521797
Jul 17 03:29:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:50.006Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-sda1.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689546526554
Jul 17 03:29:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:50.020Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"tsg_master_success_log_s\", app_name=\"tsg_master\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"traffic-engine\", line_name=\"INTERNAL-RTP-RECORD\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", table_name=\"sendlog\", vsysId=\"1\"}" timestamp=1689546525148
Jul 17 03:29:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:50.030Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-96a9f56f8880670d84ba1753f731c63c62abdde31dae3ba4a5cb169300591055-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689546526554
Jul 17 03:29:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:50.140Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"mrapm-device\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"uninterruptible\"}" timestamp=1689546514339
Jul 17 03:29:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:50.156Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_info\", asset=\"twa-tsgx10\", asset_id=\"109\", configmap=\"kubernetes-dashboard-settings\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546519668
Jul 17 03:29:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:50.158Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_usage_bytes\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/data/var/lib/kubelet/pods/a43b9675-fdba-4957-ab2b-41d084fe86c7/volumes/kubernetes.io~projected/kube-api-access-m8jml\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546522237
Jul 17 03:29:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:50.201Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_carrier_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth66a8f63b\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546526709
Jul 17 03:29:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:50.201Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"nsfs\", name=\"run-netns-cni\\\\x2d208839f0\\\\x2dcf2c\\\\x2d13e7\\\\x2d4d95\\\\x2d6092b9dffd9d.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689546521057
Jul 17 03:29:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:50.244Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-6f7082c9\\\\x2d24a1\\\\x2d4f4f\\\\x2daf11\\\\x2dba474eb9a99b-volume\\\\x2dsubpaths-config\\\\x2dvolume-telegraf\\\\x2dsecurity-2.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689546526554
Jul 17 03:29:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:50.262Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"rpc-statd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"service\"}" timestamp=1689546521057
Jul 17 03:29:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:50.264Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_gc_duration_seconds\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", quantile=\"0.5\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546527556
Jul 17 03:29:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:50.284Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_hbase_table_namespace_metric_majorCompactionInputSize_95th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546527438
Jul 17 03:29:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:50.298Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_session_record_hos_bucket_metric_majorCompactionTime_75th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546527438
Jul 17 03:29:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:50.310Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_compressed_total\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens7f1np1\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546521797
Jul 17 03:29:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:50.394Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_running\", asset=\"twa-tsgx08\", asset_id=\"105\", container=\"telegraf-security\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", pod=\"tsg-traffic-engine-numa-1-firewall-f448ffc8-zd6vt\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"6f7082c9-24a1-4f4f-af11-ba474eb9a99b\"}" timestamp=1689546521568
Jul 17 03:29:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:50.450Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_194_3_metric_majorCompactionInputFileCount_median\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546527438
Jul 17 03:29:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:50.455Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_spec_replicas\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"prometheus-kube-state-metrics-6b768c7bf8\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546521568
Jul 17 03:29:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:50.490Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"71\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546526709
Jul 17 03:29:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:50.497Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_multicast_total\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethcddab70d\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546521797
Jul 17 03:29:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:50.509Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_putCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"assessment_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"43c96dd21aa1238d6db7226daea0110e\", table=\"index_filename_assessment_hos_bucket\", type=\"index\"}" timestamp=1689546527438
Jul 17 03:29:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:50.524Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_appendCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"default\", project=\"OLAP\", project_id=\"3\", region=\"1ec6c491dc19e54814be0f747b7ceb69\", table=\"index_time_default\", type=\"index\"}" timestamp=1689546527438
Jul 17 03:29:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:50.543Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_drop_total\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth23d2a51a\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546521797
Jul 17 03:29:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:50.556Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_packets_dropped_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", interface=\"br_dign_c\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546527473
Jul 17 03:29:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:50.573Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_metadata_resource_version\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"cm-cache-extra-secrets\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546521568
Jul 17 03:29:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:50.593Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"54\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546526709
Jul 17 03:29:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:50.607Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"ext4\", name=\"var-lib-kubelet-pods-207ceb95\\\\x2de1ef\\\\x2d4b7e\\\\x2da587\\\\x2d0a6495b0f134-volume\\\\x2dsubpaths-telegraf-telegraf-0.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689546526554
Jul 17 03:29:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:50.617Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_scaling_frequency_max_hertz\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546526709
Jul 17 03:29:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:50.718Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_swap_limit_bytes\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"telegraf-security\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689546519050
Jul 17 03:29:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:50.808Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_scrape_error\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546527561
Jul 17 03:29:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:50.816Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_size_bytes\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"shm\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/3b940f5034a36f26e5912606cab9514359054bbcab4e60b30f5f852459772e99/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546521797
Jul 17 03:29:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:50.839Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_name_assign_type\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethcddab70d\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546521797
Jul 17 03:29:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:50.842Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_fully_labeled_replicas\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"traefik-64f55bb67d\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546519668
Jul 17 03:29:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:50.854Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_bytes_total\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens1f0np0\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546521797
Jul 17 03:29:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:50.864Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens3f0np0\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546521797
Jul 17 03:29:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:50.905Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_annotations\", asset=\"twa-tsgx08\", asset_id=\"105\", configmap=\"tfe-tsg-traffic-engine-numa-1\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546521568
Jul 17 03:29:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:50.922Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_info\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_endpoint=\"kube-dns\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546521568
Jul 17 03:29:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:50.962Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_bytes_total\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", interface=\"ens6f1np1\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546517654
Jul 17 03:29:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:50.975Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_created\", asset=\"twa-tsgx10\", asset_id=\"109\", configmap=\"chart-content-ipmi-exporter\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546519668
Jul 17 03:29:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:51.075Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethe417278f\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546521797
Jul 17 03:29:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:51.090Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_system_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", container=\"monit-stream\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546517755
Jul 17 03:29:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:51.123Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_packets_dropped_total\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", interface=\"ens3f0np0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546517654
Jul 17 03:29:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:51.148Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_commands_duration_seconds_total\", asset=\"twa-tsgx10\", asset_id=\"109\", cmd=\"replicaof\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546526625
Jul 17 03:29:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:51.374Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_usage_bytes\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"overlay_0-129\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546522237
Jul 17 03:29:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:51.400Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_writes_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"overlay_0-471\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546522237
Jul 17 03:29:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:51.539Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_created\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"prometheus-server\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546521568
Jul 17 03:29:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:51.564Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"twa-tsgx08\", asset_id=\"105\", container=\"kube-multus\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", failure_type=\"pgfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"hierarchy\"}" timestamp=1689546512394
Jul 17 03:29:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:51.652Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_scrape_collector_success\", asset=\"twa-tsgx08\", asset_id=\"105\", collector=\"arp\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546526709
Jul 17 03:29:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:52.640Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", cpu=\"26\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546529337
Jul 17 03:29:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:52.716Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", cpu=\"47\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546529337
Jul 17 03:29:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:52.749Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_ulimits_soft\", asset=\"pcap-tsgx11\", asset_id=\"115\", container=\"local-path-provisioner\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", ulimit=\"max_open_files\"}" timestamp=1689546523120
Jul 17 03:29:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:52.841Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"mrzcpd_hugepages_setup.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"service\"}" timestamp=1689546525496
Jul 17 03:29:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:52.901Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"overlay\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-14fcbbc790df30ece68a69014e4faab7fc423e944641982d92254e02fee996c2-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689546525496
Jul 17 03:29:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:53.047Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_labels\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-pushgateway-bd476bcf9-8t7vj\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"f2baf82c-b91b-46ec-8a59-8f55f1b86285\"}" timestamp=1689546524478
Jul 17 03:29:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:53.059Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_virtual_memory_bytes\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"rpcbind.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689546525496
Jul 17 03:29:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:53.071Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db59\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546519367
Jul 17 03:29:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:53.107Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_ulimits_soft\", asset=\"pcap-tsgx11\", asset_id=\"115\", container=\"mrapm-device\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", ulimit=\"max_open_files\"}" timestamp=1689546513808
Jul 17 03:29:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:53.126Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_cpu_seconds_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"crond.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689546525496
Jul 17 03:29:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:53.140Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-a0009b6f\\\\x2d7978\\\\x2d40ef\\\\x2da946\\\\x2dc18d7bd61697-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2d77x79.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689546525496
Jul 17 03:29:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:53.169Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-podb9a87a56_93cb_41e5_883f_d940809bc904.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"slice\"}" timestamp=1689546532135
Jul 17 03:29:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:53.182Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-f977f1abbdee782a62208d2361982507501c2fdbf749b76ef6b359a1743372f0-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689546532135
Jul 17 03:29:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:53.255Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_limit_bytes\", asset=\"msh-tsgx08\", asset_id=\"65\", container=\"exporter\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546517907
Jul 17 03:29:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:53.281Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_namespace_created\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-public\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546524478
Jul 17 03:29:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:53.328Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_replicas\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"coredns-6658b4d4d7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546524478
Jul 17 03:29:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:53.561Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", cpu=\"19\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546529337
Jul 17 03:29:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:53.584Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", container=\"exporter\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"uninterruptible\"}" timestamp=1689546527998
Jul 17 03:29:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:53.592Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-98b0a855eb12f429076c236631441411dc8b00eaab4862f1aee72ea2d6d6f9b0-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689546531179
Jul 17 03:29:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:53.607Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-user-sessions.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"service\"}" timestamp=1689546531179
Jul 17 03:29:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:53.630Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_down_changes_total\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens6f0np0\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546528328
Jul 17 03:29:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:53.648Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"msh-tsgx08\", asset_id=\"65\", cpu=\"101\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546528328
Jul 17 03:29:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:53.708Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_waiting\", asset=\"pcap-tsgx11\", asset_id=\"115\", container=\"helm\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-traefik-nhw2q\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"0c354dd1-4cfe-41af-955d-479212181025\"}" timestamp=1689546524478
Jul 17 03:29:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:53.733Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"paths.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"target\"}" timestamp=1689546532135
Jul 17 03:29:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:53.745Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"63\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546529337
Jul 17 03:29:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:53.828Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx08\", asset_id=\"65\", cpu=\"95\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546528328
Jul 17 03:29:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:53.841Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_info\", address=\"bc:97:e1:0c:04:92\", asset=\"msh-tsgx08\", asset_id=\"65\", broadcast=\"ff:ff:ff:ff:ff:ff\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens4f2np2\", duplex=\"unknown\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operstate=\"down\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546528328
Jul 17 03:29:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:53.855Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sshd-keygen@ecdsa.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"service\"}" timestamp=1689546532135
Jul 17 03:29:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:53.883Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_drop_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"eno33np0\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546519409
Jul 17 03:29:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:53.968Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_io_current\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/shm\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546516012
Jul 17 03:29:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:53.978Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx08\", asset_id=\"65\", cpu=\"80\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546528328
Jul 17 03:29:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:54.027Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"msh-tsgx08\", asset_id=\"65\", container=\"monit-stream\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"uninterruptible\"}" timestamp=1689546513038
Jul 17 03:29:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:54.096Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-dellr740-dt002\", asset_id=\"33\", cpu=\"37\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"NC-Gohangout-pcap-dellr740-dt002\", endpoint_id=\"173\", mode=\"system\", module=\"DC-Gohangout\", module_id=\"35\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546532666
Jul 17 03:29:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:54.120Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"overlay\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-d854592402e641bec71d9ffb84d68c0a24a87eb692655bf6f6e90091aa32f5b0-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689546520884
Jul 17 03:29:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:54.127Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link_mode\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens3f1np1\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546528328
Jul 17 03:29:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:54.151Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"msh-tsgx08\", asset_id=\"65\", condition=\"Progressing\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"configmap-reloader\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"true\"}" timestamp=1689546528950
Jul 17 03:29:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:54.163Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_flags\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth35760bab\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546523391
Jul 17 03:29:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:54.164Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_memory_SwapCached_bytes\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546528328
Jul 17 03:29:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:54.181Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:20-0000:20:03.1-0000:21:00.0-infiniband-mlx5_2.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689546520884
Jul 17 03:29:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:54.199Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", container=\"dign-server-web\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Total\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546527080
Jul 17 03:29:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:54.216Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Succeeded\", pod=\"helm-install-mrzcpd-exporter-4rjwg\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"9b1ef315-33f8-4a3d-a750-cb303d9afe8c\"}" timestamp=1689546528950
Jul 17 03:29:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:54.234Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", container=\"telegraf\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"cadvisor\", failure_type=\"pgmajfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"container\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689546518974
Jul 17 03:29:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:54.253Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"75\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546519409
Jul 17 03:29:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:54.272Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-firstboot.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"service\"}" timestamp=1689546532135
Jul 17 03:29:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:54.279Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_bytes_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"cadvisor\", interface=\"ens7f1np1\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546528371
Jul 17 03:29:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:54.306Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"msh-tsgx12\", asset_id=\"113\", container=\"mrapm-device\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"stopped\"}" timestamp=1689546509535
Jul 17 03:29:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:54.336Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_info\", asset=\"msh-tsgx08\", asset_id=\"65\", container=\"mrapm-device\", container_id=\"containerd://4219f57df493e82093b441fc6fcd13b4ccb880c86ff97b516fa06933f0f5f4a0\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", image=\"registry.gdnt-cloud.website/mrzcpd-exporter:1.0.1\", image_id=\"sha256:b1a7adfb61be24490a1321c37405d86204b78b1ee251ed959d48dc357310fb1c\", image_spec=\"registry.gdnt-cloud.website/mrzcpd-exporter:1.0.1\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"mrzcpd-exporter-5cf7f967cc-m79nw\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"2c4890df-bfb7-48e8-a274-494cc0f3691d\"}" timestamp=1689546528950
Jul 17 03:29:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:54.337Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_timex_pps_jitter_seconds\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546519409
Jul 17 03:29:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:54.404Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db11\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546525184
Jul 17 03:29:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:54.497Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_spec_replicas\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"cm-policy-local-cache-redis-master\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546522058
Jul 17 03:29:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:54.566Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-configmap-reloader-mhdtn\", project=\"TSG-OS\", project_id=\"7\", reason=\"Evicted\", service=\"prometheus-kube-state-metrics\", uid=\"9aa69a60-f093-4b03-8dd0-aa1af3f43f56\"}" timestamp=1689546528663
Jul 17 03:29:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:54.599Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda5\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", fstype=\"ext4\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/var/lib/kubelet/pods/8ba148c1-f25f-4fa4-82b3-03b9d5a8ace6/volume-subpaths/config-volume/telegraf-security/2\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546523391
Jul 17 03:29:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:54.627Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_packets_received_total\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_10\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546523391
Jul 17 03:29:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:54.651Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_daemonset_labels\", asset=\"pcap-tsgx07\", asset_id=\"85\", daemonset=\"dign-server\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546522058
Jul 17 03:29:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:54.694Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cpu=\"121\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546519409
Jul 17 03:29:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:54.764Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-30e825a7b45908e015280eb0f8420785cf4c214ceaa2a725354ab6baa5bad742-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689546531179
Jul 17 03:29:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:54.929Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_node_status_condition\", asset=\"pcap-tsgx07\", asset_id=\"85\", condition=\"PIDPressure\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", node=\"localhost\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"true\"}" timestamp=1689546522058
Jul 17 03:29:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:54.948Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db117\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546518659
Jul 17 03:29:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:54.986Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-f153cf9453d845f0ebe6be6d4a8de54acb6df2e77eaba09a037dfa8db12fdc24.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"scope\"}" timestamp=1689546531179
Jul 17 03:29:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:55.051Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-ttyS0.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689546520884
Jul 17 03:29:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:55.078Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"nfsdcld.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689546520884
Jul 17 03:29:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:55.190Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_metadata_generation\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"mrzcpd-exporter\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546522058
Jul 17 03:29:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:55.414Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_name_assign_type\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"cnibr0\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546519409
Jul 17 03:29:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:56.276Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_compressed_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth58e91056\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546525883
Jul 17 03:29:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:56.291Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_ips\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", ip=\"10.42.0.115\", ip_family=\"4\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-pushgateway-bd476bcf9-drbj7\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"1f479ae1-34c9-42c8-b82d-e0af461da2f5\"}" timestamp=1689546533389
Jul 17 03:29:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:56.305Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"15\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546525883
Jul 17 03:29:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:56.376Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_threads\", asset=\"pcap-tsgx03\", asset_id=\"73\", container=\"redis\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546519086
Jul 17 03:29:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:56.393Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_spec_parallelism\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", job_name=\"helm-install-systemd-exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546533389
Jul 17 03:29:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:56.415Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"1\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546525883
Jul 17 03:29:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:57.352Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_packets_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"cadvisor\", interface=\"ens6f0np0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546531104
Jul 17 03:29:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:57.366Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_created\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", job_name=\"helm-install-traefik-crd\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546533389
Jul 17 03:29:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:57.502Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_working_set_bytes\", asset=\"pcap-tsgx03\", asset_id=\"73\", container=\"kube-multus\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546521046
Jul 17 03:29:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:57.679Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_free_bytes\", asset=\"msh-dellr740-dt003\", asset_id=\"9\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sdc1\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-dt003\", endpoint_id=\"229\", fstype=\"xfs\", module=\"OLAP-Node-Exporter\", module_id=\"51\", mountpoint=\"/data\", nz_agent_id=\"3\", olap_node_exporter=\"hbase_master,clickhouse_data,kafka,gohangout,zookeeper,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546537407
Jul 17 03:29:58 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:58.080Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_scaling_frequency_max_hertz\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", cpu=\"35\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"OLAP-Node-Exporter-pcap-dellr740-dt001\", endpoint_id=\"251\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546537557
Jul 17 03:29:58 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:58.114Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", cpu=\"21\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"OLAP-Node-Exporter-pcap-dellr740-dt001\", endpoint_id=\"251\", mode=\"system\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546537557
Jul 17 03:29:58 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:58.144Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_address_assign_type\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"em3\", endpoint=\"OLAP-Node-Exporter-pcap-dellr740-dt001\", endpoint_id=\"251\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546537557
Jul 17 03:29:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:59.904Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"89\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546529083
Jul 17 03:29:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:29:59.922Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db31\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546525071
Jul 17 03:30:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:00.029Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_annotations\", asset=\"msh-tsgx01\", asset_id=\"51\", configmap=\"kube-root-ca.crt\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546527425
Jul 17 03:30:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:00.038Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"124\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546529083
Jul 17 03:30:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:00.054Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546529083
Jul 17 03:30:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:00.209Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:c0-0000:c0:01.1-0000:c1:00.1-net-ens7f1np1.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689546524879
Jul 17 03:30:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:00.291Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_commands_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cmd=\"get\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546525071
Jul 17 03:30:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:00.354Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_rss\", asset=\"msh-tsgx01\", asset_id=\"51\", container=\"prometheus-server-configmap-reload\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546508064
Jul 17 03:30:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:00.410Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", phase=\"Succeeded\", pod=\"prometheus-kube-state-metrics-6b768c7bf8-fnk7m\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"75a1cd69-272c-4ee3-afeb-f27cbbc02353\"}" timestamp=1689546527425
Jul 17 03:30:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:00.529Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"102\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546529083
Jul 17 03:30:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:00.572Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"msh-tsgx01\", asset_id=\"51\", condition=\"Progressing\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"cm-policy-local-cache-redis-master\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"true\"}" timestamp=1689546527425
Jul 17 03:30:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:00.586Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_storageclass_created\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", storageclass=\"local-path\"}" timestamp=1689546527425
Jul 17 03:30:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:00.621Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_errors_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"cadvisor\", interface=\"ens4f0np0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546532897
Jul 17 03:30:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:00.700Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"hasp_monitor.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"service\"}" timestamp=1689546524879
Jul 17 03:30:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:00.847Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_info\", address=\"b8:ce:f6:72:b1:58\", asset=\"msh-tsgx01\", asset_id=\"51\", broadcast=\"ff:ff:ff:ff:ff:ff\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens2f0np0\", duplex=\"unknown\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operstate=\"down\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546529083
Jul 17 03:30:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:01.182Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_created\", asset=\"msh-tsgx01\", asset_id=\"51\", configmap=\"kube-apiserver-legacy-service-account-token-tracking\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546527425
Jul 17 03:30:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:01.215Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Running\", pod=\"helm-install-systemd-exporter-vtkqz\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"de1fecec-5049-4373-b10d-2d1c756dd25d\"}" timestamp=1689546527425
Jul 17 03:30:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:02.579Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_working_set_bytes\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546531550
Jul 17 03:30:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:02.788Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_disk_io_time_seconds_total\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"sda\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546538318
Jul 17 03:30:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:02.855Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_ready_replicas\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"coredns-858dc96bbf\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546531412
Jul 17 03:30:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:02.887Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_status_completion_time\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", job_name=\"helm-install-systemd-exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546531412
Jul 17 03:30:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:02.934Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_errs_total\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens1f0np0\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546538318
Jul 17 03:30:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:02.947Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-19e626095d3782c6fc90a46b8fca2ae0d75e7c10299930eba0a129115923ef3a.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"scope\"}" timestamp=1689546528447
Jul 17 03:30:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:02.949Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_labels\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-alertmanager-6df7f457b6-tk42l\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"b4a34040-e633-4749-bfb3-b03089d307fe\"}" timestamp=1689546531412
Jul 17 03:30:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:03.304Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"net_conntrack_dialer_conn_failed_total\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", dialer_name=\"asset_ping_69\", instance=\"10.10.20.159:10090\", reason=\"resolution\", type=\"global\"}" timestamp=1689546542990
Jul 17 03:30:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:03.578Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"net_conntrack_dialer_conn_failed_total\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", dialer_name=\"161\", instance=\"10.10.20.159:10090\", reason=\"resolution\", type=\"global\"}" timestamp=1689546542990
Jul 17 03:30:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:03.710Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_threads_max\", asset=\"msh-tsgx02\", asset_id=\"63\", container=\"exporter-merger\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546531357
Jul 17 03:30:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:03.755Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"net_conntrack_dialer_conn_failed_total\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", dialer_name=\"201\", instance=\"10.10.20.159:10090\", reason=\"refused\", type=\"global\"}" timestamp=1689546542990
Jul 17 03:30:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:03.912Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_NameNode_PercentUsed\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-Hadoop-msh-dellr740-dt001-NameNode\", endpoint_id=\"327\", module=\"DC-Hadoop\", module_id=\"53\", name=\"NameNodeInfo\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546543189
Jul 17 03:30:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:04.221Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-rdma-devices-mlx5_2:1-umad.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689546528447
Jul 17 03:30:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:04.583Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx02\", asset_id=\"63\", cpu=\"24\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546538318
Jul 17 03:30:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:04.598Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-23832833f719a564a8d2b52445b2f8883c2a8dbb929653e9fd02068486a6e87f-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689546528447
Jul 17 03:30:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:04.624Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"prometheus_target_scrape_pool_sync_total\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", instance=\"10.10.20.159:10090\", scrape_job=\"319\", type=\"global\"}" timestamp=1689546542990
Jul 17 03:30:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:04.744Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_latest_fork_seconds\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546529269
Jul 17 03:30:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:04.802Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_core_throttles_total\", asset=\"msh-dellr740-olap005\", asset_id=\"19\", core=\"3\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Spark-msh-dellr740-olap005\", endpoint_id=\"17\", module=\"NC-Spark\", module_id=\"47\", nz_agent_id=\"3\", package=\"1\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546543393
Jul 17 03:30:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:04.887Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx02\", asset_id=\"63\", cpu=\"83\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546538318
Jul 17 03:30:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:04.901Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"tmpfs\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/var/lib/kubelet/pods/95d4eb15-f514-4daa-ac28-6317addaafb6/volumes/kubernetes.io~secret/config\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546538318
Jul 17 03:30:05 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:05.138Z caller=dedupe.go:112 component=remote level=info remote_name=52db6f url=http://10.10.10.159:10090/nz-agent/prometheus/proxy/api/v1/write msg="Dropped sample for series that was not explicitly dropped via relabelling" ref=4436650
Jul 17 03:30:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:06.607Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cpu=\"23\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546543910
Jul 17 03:30:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:06.636Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_errors_total\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"cadvisor\", interface=\"ens1f0np0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546539505
Jul 17 03:30:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:06.639Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_ready\", asset=\"twa-tsgx01\", asset_id=\"91\", container=\"helm\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-tsg-traffic-engine-numa-1-nzg5m\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"f36b39da-a7f4-44d5-a1b0-fc87a0caf7ea\"}" timestamp=1689546541227
Jul 17 03:30:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:06.702Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_local_link_integrity_errors_total\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_10\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546543910
Jul 17 03:30:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:06.743Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_fifo_total\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"bond0\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546543910
Jul 17 03:30:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:06.884Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_address_assign_type\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens6f0np0\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546543910
Jul 17 03:30:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:06.998Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_drop_total\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth747cb07e\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546543910
Jul 17 03:30:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:07.096Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2dpath-pci\\\\x2d0000:81:00.0\\\\x2dscsi\\\\x2d0:2:0:0\\\\x2dpart2.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689546533582
Jul 17 03:30:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:07.390Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_writes_total\", asset=\"twa-tsgx01\", asset_id=\"91\", container=\"dashboard-metrics-scraper\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546536698
Jul 17 03:30:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:07.405Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cpu=\"79\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546543910
Jul 17 03:30:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:07.437Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_excessive_buffer_overrun_errors_total\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_4\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546543910
Jul 17 03:30:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:07.473Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_mod_info\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"github.com/prometheus/common\", nz_agent_id=\"3\", program=\"systemd_exporter\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", version=\"v0.9.1\"}" timestamp=1689546533582
Jul 17 03:30:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:07.491Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_restarts_total\", asset=\"twa-tsgx01\", asset_id=\"91\", container=\"local-path-provisioner\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"local-path-provisioner-957fdf8bc-gd58k\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"d2cea37b-373b-49b2-8c41-c9223bc419df\"}" timestamp=1689546541227
Jul 17 03:30:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:07.506Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cpu=\"4\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546543910
Jul 17 03:30:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:07.533Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_annotations\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"localhost.node-password.k3s\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546541227
Jul 17 03:30:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:07.536Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_limit_bytes\", asset=\"twa-tsgx01\", asset_id=\"91\", container=\"redis\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546542966
Jul 17 03:30:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:07.661Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_mapped_file\", asset=\"twa-tsgx01\", asset_id=\"91\", container=\"lb-tcp-80\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546541128
Jul 17 03:30:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:07.680Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-mqueue.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689546533582
Jul 17 03:30:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:07.902Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_virtual_memory_bytes\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"tsg-clixon-backend.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689546533582
Jul 17 03:30:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:07.967Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"blk-availability.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"oneshot\", type=\"service\"}" timestamp=1689546533582
Jul 17 03:30:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:08.045Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"rc-local.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689546533582
Jul 17 03:30:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:08.070Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_working_set_bytes\", asset=\"twa-tsgx01\", asset_id=\"91\", container=\"kubernetes-dashboard\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546540000
Jul 17 03:30:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:08.131Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_state_started\", asset=\"twa-tsgx01\", asset_id=\"91\", container=\"traefik\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"traefik-64f55bb67d-bd6jb\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"377e59e7-1c6e-433a-a40c-92bbd7c96064\"}" timestamp=1689546541227
Jul 17 03:30:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:08.158Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"110\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546543910
Jul 17 03:30:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:10.565Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_getCount\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"rtp_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"4f1f29963253708c1bdc3f4258297501\", table=\"index_partfile_rtp_hos_bucket\", type=\"index\"}" timestamp=1689546548805
Jul 17 03:30:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:11.032Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_avgStoreFileAge\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"knowledge_base_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"8f77301947b837805e9b5a6254c5a226\", table=\"index_time_knowledge_base_hos_bucket\", type=\"index\"}" timestamp=1689546548805
Jul 17 03:30:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:11.150Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_deleteCount\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"d04540053f8ee2bdfc50c56870726386\", table=\"proxy_hos_bucket\", type=\"data\"}" timestamp=1689546548805
Jul 17 03:30:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:11.521Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_STATS_metric_writeRequestCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546550148
Jul 17 03:30:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:11.553Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_4_metric_putBatchTime_99_9th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546550148
Jul 17 03:30:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:12.051Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_session_record_hos_bucket_metric_flushMemstoreSize_min\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546550148
Jul 17 03:30:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:12.360Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_filteredReadRequestCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"assessment_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"95b99327857538457c532139e74048fd\", table=\"index_partfile_assessment_hos_bucket\", type=\"index\"}" timestamp=1689546550148
Jul 17 03:30:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:12.547Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_flushesQueuedCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"proxy_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"acbd290c0645e422c9b3249c06c768ab\", table=\"index_filename_proxy_hos_bucket\", type=\"index\"}" timestamp=1689546550148
Jul 17 03:30:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:12.606Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_minStoreFileAge\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"268471f048bc85ce60683d08da793a10\", table=\"assessment_hos_bucket\", type=\"data\"}" timestamp=1689546550148
Jul 17 03:30:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:12.663Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_3_metric_compactionInputFileCount_25th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546550148
Jul 17 03:30:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:12.711Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_3_metric_majorCompactionOutputFileCount_99th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546550148
Jul 17 03:30:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:12.887Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_storeCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", namespace=\"SYSTEM\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"b0d947844d6d153d8ca0086e16920347\", table=\"LOG\", type=\"data\"}" timestamp=1689546550148
Jul 17 03:30:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:12.973Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_194_3_metric_deleteBatchTime_90th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546550148
Jul 17 03:30:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:15.435Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_terminated\", asset=\"twa-tsgx07\", asset_id=\"103\", container=\"exporter\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"systemd-exporter-7f549d9485-jkp9n\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"c435f080-d1a5-46a3-868f-b5d8209ce2ea\"}" timestamp=1689546548076
Jul 17 03:30:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:15.782Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_persistentvolumeclaim_status_phase\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", persistentvolumeclaim=\"prometheus-alertmanager\", phase=\"Lost\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546548076
Jul 17 03:30:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:15.858Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"chronyd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"service\"}" timestamp=1689546547046
Jul 17 03:30:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:15.881Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2duuid-0e9caa87\\\\x2d4353\\\\x2d41b2\\\\x2d8439\\\\x2d7d8fdb38298e.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689546547046
Jul 17 03:30:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:16.137Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_annotations\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"chart-values-tsg-traffic-engine-numa-0\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546548076
Jul 17 03:30:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:16.157Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"127\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546541230
Jul 17 03:30:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:16.170Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db6\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546546027
Jul 17 03:30:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:16.306Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_errors_received_total\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_9\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546541230
Jul 17 03:30:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:16.404Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_load_average_10s\", asset=\"twa-tsgx07\", asset_id=\"103\", container=\"lb-tcp-80\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546522582
Jul 17 03:30:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:16.423Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", pod=\"tsg-traffic-engine-numa-1-firewall-574b9bd44b-zm8zv\", project=\"TSG-OS\", project_id=\"7\", reason=\"NodeLost\", service=\"prometheus-kube-state-metrics\", uid=\"704b5479-b3a5-4f40-a111-190391574a21\"}" timestamp=1689546548076
Jul 17 03:30:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:16.437Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_speed_bytes\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth5c296dad\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546541230
Jul 17 03:30:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:16.492Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_ready\", asset=\"twa-tsgx07\", asset_id=\"103\", container=\"firewall\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", pod=\"tsg-traffic-engine-numa-0-firewall-5b47889f4f-8qqtj\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"64082a69-3be2-4b5e-9e3a-c0da16576d9e\"}" timestamp=1689546548076
Jul 17 03:30:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:16.506Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_running\", asset=\"twa-tsgx07\", asset_id=\"103\", container=\"prometheus-server-configmap-reload\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-server-6ff784d758-sgqd4\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"0ed53af9-6e57-4bfe-bb64-b0aa4edce9c5\"}" timestamp=1689546548076
Jul 17 03:30:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:16.550Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546541230
Jul 17 03:30:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:16.808Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"tmpfs\", name=\"run-k3s-containerd-io.containerd.grpc.v1.cri-sandboxes-1e88ab467eb5d433e3551ba834ec5d53423871458d51d9a6c91448ff517f5abf-shm.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689546547046
Jul 17 03:30:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:16.858Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_labels\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.mrzcpd-exporter.v1\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546548076
Jul 17 03:30:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:16.873Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"shm\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/2e122063ee2c201eace04f25ac4f6d02502e3258d1a3613479961720dee5e2b1/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546541230
Jul 17 03:30:17 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:17.406Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_NodeManager_MemNonHeapUsedM\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-Yarn-pcap-dellr740-dt001-NodeManager\", endpoint_id=\"357\", module=\"DC-Yarn\", module_id=\"1\", name=\"JvmMetrics\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546557154
Jul 17 03:30:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:19.493Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"124\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546557612
Jul 17 03:30:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:19.594Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"var-lib-kubelet-pods-fe59f4a3\\\\x2dab44\\\\x2d46a4\\\\x2d9b1d\\\\x2d4a02aedb6f14-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2dpxqk9.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689546544356
Jul 17 03:30:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:19.602Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"80\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546557612
Jul 17 03:30:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:19.631Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_name_assign_type\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethef8c731a\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546557612
Jul 17 03:30:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:19.679Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"67\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546557612
Jul 17 03:30:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:19.721Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx06\", asset_id=\"101\", container=\"dign-server-web\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Total\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546548911
Jul 17 03:30:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:19.730Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_sockets\", asset=\"twa-tsgx06\", asset_id=\"101\", container=\"dign-client\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546548204
Jul 17 03:30:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:19.766Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_observed_generation\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-1-firewall-6b6bf4679c\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546553949
Jul 17 03:30:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:19.777Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx06\", asset_id=\"101\", container=\"dign-client\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Total\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546548204
Jul 17 03:30:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:19.783Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_metadata_resource_version\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"cm-cache-extra-secrets\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546553949
Jul 17 03:30:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:19.796Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_info\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_endpoint=\"cm-policy-local-cache-redis-metrics\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546553949
Jul 17 03:30:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:19.802Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"34\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546557612
Jul 17 03:30:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:19.811Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_max_usage_bytes\", asset=\"twa-tsgx06\", asset_id=\"101\", container=\"firewall\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689546552340
Jul 17 03:30:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:19.826Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_bytes_total\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens4f3np3\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546557612
Jul 17 03:30:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:20.057Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"twa-tsgx06\", asset_id=\"101\", container=\"ipmi-exporter\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"iowaiting\"}" timestamp=1689546549457
Jul 17 03:30:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:20.091Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_working_set_bytes\", asset=\"twa-tsgx06\", asset_id=\"101\", container=\"mrapm-device\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546551027
Jul 17 03:30:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:20.109Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_spec_parallelism\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", job_name=\"helm-install-tsg-traffic-engine-numa-0\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546553949
Jul 17 03:30:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:20.192Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"127\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546557612
Jul 17 03:30:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:20.322Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_name_assign_type\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens4f1np1\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546557612
Jul 17 03:30:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:20.421Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"47\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546548263
Jul 17 03:30:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:20.439Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db233\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546554840
Jul 17 03:30:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:20.457Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-c5b2ed12\\\\x2d454d\\\\x2d4011\\\\x2d8425\\\\x2d1b76bb8c99c4-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2dmxdqs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689546550657
Jul 17 03:30:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:20.510Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-sda3.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689546550657
Jul 17 03:30:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:20.519Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-sda4.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689546550657
Jul 17 03:30:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:20.525Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"98\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546557612
Jul 17 03:30:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:20.539Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"15\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546557612
Jul 17 03:30:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:20.540Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"85\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546557612
Jul 17 03:30:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:20.567Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:a0-0000:a0:03.1-0000:a1:00.0-infiniband_mad-umad6.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689546550657
Jul 17 03:30:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:20.622Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"iscsi-onboot.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"service\"}" timestamp=1689546544356
Jul 17 03:30:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:20.701Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_completion_time\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-tsg-traffic-engine-numa-0-sfc4c\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"02e61820-d9ba-4a99-8e7c-ef3b109dc48f\"}" timestamp=1689546553949
Jul 17 03:30:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:20.728Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", phase=\"Unknown\", pod=\"dign-client-llnpb\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"78246b9b-6c2f-4139-8e55-13e52335743d\"}" timestamp=1689546553949
Jul 17 03:30:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:20.796Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"75\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546557612
Jul 17 03:30:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:20.812Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_unicast_packets_transmitted_total\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_1\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546557612
Jul 17 03:30:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:20.834Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546557612
Jul 17 03:30:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:20.850Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"106\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546557612
Jul 17 03:30:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:20.888Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_ulimits_soft\", asset=\"twa-tsgx06\", asset_id=\"101\", container=\"prometheus-pushgateway\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", ulimit=\"max_open_files\"}" timestamp=1689546547602
Jul 17 03:30:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:20.914Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_complete\", asset=\"twa-tsgx06\", asset_id=\"101\", condition=\"true\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", job_name=\"helm-install-tsg-traffic-engine-numa-1\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546553949
Jul 17 03:30:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:20.931Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_multicast_packets_transmitted_total\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_11\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546557612
Jul 17 03:30:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:20.967Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_drop_total\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth809b20ca\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546557612
Jul 17 03:30:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:21.001Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2did-scsi\\\\x2dSDELL_PERC_H345_Adp_0046807ed21cf8262c00758284f0a7ce.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689546544356
Jul 17 03:30:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:21.019Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-21a8f725e0c245f8b4fe816ffaa75ab509a037d587c94c5dc2716994c55e4176.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"scope\"}" timestamp=1689546544356
Jul 17 03:30:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:21.047Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"fcoe.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"service\"}" timestamp=1689546544356
Jul 17 03:30:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:21.248Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-617272f1\\\\x2de26a\\\\x2d4d41\\\\x2db763\\\\x2d554ea7e04473-volume\\\\x2dsubpaths-telegraf-telegraf-0.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689546550657
Jul 17 03:30:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:21.376Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"initrd-root-device.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"target\"}" timestamp=1689546550657
Jul 17 03:30:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:21.396Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_created\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.tsg-traffic-engine-numa-0.v3\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546553828
Jul 17 03:30:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:21.518Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_labels\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-master\", uid=\"34339d0f-dc05-4c2c-9fa5-b9156418cc9f\"}" timestamp=1689546553828
Jul 17 03:30:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:21.838Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"pcap-tsgx10\", asset_id=\"89\", condition=\"false\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"kube-multus-ds-sqrb7\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"ee8afe42-b7fb-4840-a9ee-3d0e7212c356\"}" timestamp=1689546553828
Jul 17 03:30:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:21.882Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens4f0np0\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546548263
Jul 17 03:30:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:21.987Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_colls_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth86dfd1bd\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546548263
Jul 17 03:30:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:22.097Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_mspan_sys_bytes\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546554840
Jul 17 03:30:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:22.166Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-b716684eb3dce8b557b9edeb3539f1c8d8445bf3a2d2d76ebfe05808d7eb2804-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689546550657
Jul 17 03:30:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:22.225Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_replicas\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"mrzcpd-exporter-5cf7f967cc\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546553828
Jul 17 03:30:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:22.260Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_write_seconds_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda2\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546553879
Jul 17 03:30:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:22.305Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", container=\"exporter-merger\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Async\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546551515
Jul 17 03:30:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:22.324Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_mspan_inuse_bytes\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546554840
Jul 17 03:30:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:22.552Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"java_nio_BufferPool_Count\", asset=\"msh-dellr740-dt003\", asset_id=\"9\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-Yarn-msh-dellr740-dt003-NodeManager\", endpoint_id=\"355\", module=\"DC-Yarn\", module_id=\"1\", name=\"direct\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546562405
Jul 17 03:30:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:23.639Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"overlay\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-2b50902878f015e6048315cd7747aafba261f013bc1962873baf8c64727ad42e-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689546558416
Jul 17 03:30:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:23.659Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"nfs-utils.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689546558416
Jul 17 03:30:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:23.677Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-70c311b2ab55787e217d750eb890fd9b9301c05719bfb93f9ef9f98d4b4f36b7-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689546558416
Jul 17 03:30:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:23.739Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"msh-tsgx09\", asset_id=\"55\", container=\"prometheus-server\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"sleeping\"}" timestamp=1689546547765
Jul 17 03:30:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:23.791Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"msh-tsgx09\", asset_id=\"55\", condition=\"true\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-traefik-crd-tjh7p\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"2a496308-3fd5-494e-83f9-537bb34e5898\"}" timestamp=1689546560068
Jul 17 03:30:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:23.805Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-f2551d6b\\\\x2d0939\\\\x2d434d\\\\x2dba32\\\\x2d2e369c56471c-volume\\\\x2dsubpaths-config\\\\x2dvolume-firewall-1.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689546558416
Jul 17 03:30:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:23.805Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"ldconfig.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"oneshot\", type=\"service\"}" timestamp=1689546558416
Jul 17 03:30:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:23.862Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-852b631a3baff6085e592e514448bfb1292ec41588c97b9cc1e9c05320126a83.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"scope\"}" timestamp=1689546558416
Jul 17 03:30:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:23.881Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"k3s.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"service\"}" timestamp=1689546558416
Jul 17 03:30:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:24.002Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dracut-pre-mount.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"oneshot\", type=\"service\"}" timestamp=1689546558416
Jul 17 03:30:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:24.305Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"57\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546554790
Jul 17 03:30:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:24.337Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_info\", address=\"b8:ce:f6:29:94:a7\", asset=\"msh-tsgx09\", asset_id=\"55\", broadcast=\"ff:ff:ff:ff:ff:ff\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens3f1np1\", duplex=\"unknown\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operstate=\"down\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546554790
Jul 17 03:30:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:24.390Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-0632ec2f152e4f21e124df9c303633ee127dc67fc3bcb7eab93790c3f911033a.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"scope\"}" timestamp=1689546558416
Jul 17 03:30:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:24.413Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"snmptrapd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"service\"}" timestamp=1689546558416
Jul 17 03:30:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:24.485Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"overlay\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-e29ba7f3c6e483ee504e55d39207f8bc66701a3ab2cd81ede609c6ee5b1d703d-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689546558416
Jul 17 03:30:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:24.640Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"36\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546554790
Jul 17 03:30:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:24.833Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_threads\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"firewall\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689546553428
Jul 17 03:30:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:24.969Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_down_changes_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens7f1np1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546560160
Jul 17 03:30:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:24.997Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-e5d1d10290add0d5022101c67a41ccb9c7da13f2242f7941ff3f3b8fc1071644.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"scope\"}" timestamp=1689546558416
Jul 17 03:30:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:25.016Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_fifo_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens1f1np1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546560160
Jul 17 03:30:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:25.499Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_errs_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens4f0np0\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546560160
Jul 17 03:30:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:25.694Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-tmpfiles-clean.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"service\"}" timestamp=1689546551102
Jul 17 03:30:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:25.758Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_speed_bytes\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth7ba955f4\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546560160
Jul 17 03:30:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:25.928Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"nginx_vts_upstream_request_seconds_total\", asset=\"msh-dellr740-olap001\", asset_id=\"11\", backend=\"192.168.20.222:8183\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"Galaxy-gateway-nginx-msh-dellr740-olap001\", endpoint_id=\"43\", module=\"Galaxy-gateway-nginx\", module_id=\"11\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", upstream=\"qgwService\"}" timestamp=1689546564440
Jul 17 03:30:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:25.932Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_mtu_bytes\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"virtio_dign_s\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546560160
Jul 17 03:30:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:25.948Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db49\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546561795
Jul 17 03:30:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:26.007Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"101\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546560160
Jul 17 03:30:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:26.027Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_mtu_bytes\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"vethf6db1ea3\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546560160
Jul 17 03:30:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:26.046Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx09\", asset_id=\"107\", container=\"dign-server-dns\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Async\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546552641
Jul 17 03:30:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:26.059Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"51\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546555810
Jul 17 03:30:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:26.071Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_cpu_period\", asset=\"twa-tsgx09\", asset_id=\"107\", container=\"exporter\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546565490
Jul 17 03:30:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:26.086Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_writes_bytes_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"configmap-reloader\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546561029
Jul 17 03:30:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:26.121Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"twa-tsgx09\", asset_id=\"107\", condition=\"false\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-server-6ff784d758-mtn8x\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"55c0a8e2-51a8-48bf-a3cc-51508b7d7254\"}" timestamp=1689546558744
Jul 17 03:30:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:26.129Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"systemd-exporter-7f549d9485-8vz6g\", project=\"TSG-OS\", project_id=\"7\", reason=\"UnexpectedAdmissionError\", service=\"prometheus-kube-state-metrics\", uid=\"0fb77c5e-cf94-4f7b-a373-4624f04e325b\"}" timestamp=1689546558744
Jul 17 03:30:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:26.151Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"27\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546555810
Jul 17 03:30:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:26.169Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_scrape_collector_success\", asset=\"twa-tsgx09\", asset_id=\"107\", collector=\"nfsd\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546555810
Jul 17 03:30:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:26.324Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_memory_used_bytes\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546563549
Jul 17 03:30:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:26.331Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_spec_replicas\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"local-path-provisioner\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546558744
Jul 17 03:30:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:26.345Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"75\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546555810
Jul 17 03:30:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:26.364Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_persistentvolume_capacity_bytes\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", persistentvolume=\"pvc-d87f51a9-8749-4943-8cd8-726e2278e14a\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546558744
Jul 17 03:30:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:26.465Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_rss\", asset=\"twa-tsgx09\", asset_id=\"107\", container=\"coredns\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546561821
Jul 17 03:30:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:26.496Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"http_server_requests_seconds_sum\", asset=\"msh-dellr740-olap001\", asset_id=\"11\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Nacos-msh-dellr740-olap001\", endpoint_id=\"277\", exception=\"None\", method=\"POST\", module=\"NC-Nacos\", module_id=\"45\", nz_agent_id=\"3\", outcome=\"SUCCESS\", project=\"OLAP\", project_id=\"3\", status=\"200\", uri=\"/v1/ns/raft/beat\"}" timestamp=1689546565919
Jul 17 03:30:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:26.616Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"exporter-systemd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"service\"}" timestamp=1689546553907
Jul 17 03:30:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:26.630Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"rpc-statd-notify.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"service\"}" timestamp=1689546553907
Jul 17 03:30:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:26.653Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_max_usage_bytes\", asset=\"twa-tsgx09\", asset_id=\"107\", container=\"redis\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546555815
Jul 17 03:30:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:26.671Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"92\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546555810
Jul 17 03:30:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:26.706Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"basic.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"target\"}" timestamp=1689546553907
Jul 17 03:30:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:26.747Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"libstoragemgmt.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"service\"}" timestamp=1689546553907
Jul 17 03:30:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:26.813Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_errs_total\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth0fc63319\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546555810
Jul 17 03:30:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:26.968Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_max_usage_bytes\", asset=\"twa-tsgx09\", asset_id=\"107\", container=\"traefik\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546560843
Jul 17 03:30:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:26.983Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_daemonset_status_desired_number_scheduled\", asset=\"twa-tsgx09\", asset_id=\"107\", daemonset=\"dign-client\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546558744
Jul 17 03:30:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:27.005Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"jvm_memory_max_bytes\", area=\"heap\", asset=\"msh-dellr740-olap001\", asset_id=\"11\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Nacos-msh-dellr740-olap001\", endpoint_id=\"277\", id=\"PS Survivor Space\", module=\"NC-Nacos\", module_id=\"45\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546565919
Jul 17 03:30:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:27.023Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_processes\", asset=\"twa-tsgx09\", asset_id=\"107\", container=\"telegraf\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689546560765
Jul 17 03:30:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:27.049Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"nsfs\", name=\"run-netns-cni\\\\x2dd57a4f50\\\\x2dd662\\\\x2d52ed\\\\x2d4a07\\\\x2d028e1b6d7ac6.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689546553907
Jul 17 03:30:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:27.113Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"twa-tsgx09\", asset_id=\"107\", container=\"prometheus-pushgateway\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"uninterruptible\"}" timestamp=1689546558718
Jul 17 03:30:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:27.126Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"tmpfs\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/data/var/lib/kubelet/pods/680c59bb-c7ab-427d-b652-0e744860b375/volumes/kubernetes.io~projected/kube-api-access-svt2c\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546555810
Jul 17 03:30:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:27.131Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_created\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.tsg-traffic-engine-numa-1.v2\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546558744
Jul 17 03:30:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:27.140Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-virtual-net-veth4c7179cd.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689546553907
Jul 17 03:30:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:27.151Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"44\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546555810
Jul 17 03:30:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:27.309Z caller=dedupe.go:112 component=remote level=error remote_name=52db6f url=http://10.10.10.159:10090/nz-agent/prometheus/proxy/api/v1/write msg="non-recoverable error" count=1000 exemplarCount=0 err="server returned HTTP status 400 : out of order sample"
Jul 17 03:30:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:27.502Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_files\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", fstype=\"ext4\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/update\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546562882
Jul 17 03:30:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:27.629Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"msh-tsgx10\", asset_id=\"69\", condition=\"Progressing\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"mrzcpd-exporter\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"true\"}" timestamp=1689546555682
Jul 17 03:30:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:27.801Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_annotations\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-configmap-reloader-szrcq\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"99814cb0-81f7-4fa6-9f8c-54016e6c8558\"}" timestamp=1689546555682
Jul 17 03:30:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:27.976Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"rescue.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"service\"}" timestamp=1689546554114
Jul 17 03:30:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:28.020Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_start_time\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"kube-multus-ds-ntb9l\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"396cb6af-9b76-4a32-aad2-deb6dbe16f5b\"}" timestamp=1689546555682
Jul 17 03:30:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:28.035Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"coredns\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Sync\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546546933
Jul 17 03:30:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:28.058Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_completion_time\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-traefik-crd-m2nsh\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"6ee7b6d8-27b2-4118-9cf0-9229465d0b5b\"}" timestamp=1689546555682
Jul 17 03:30:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:28.097Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"62\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546562882
Jul 17 03:30:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:28.118Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"rsyslog.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689546554114
Jul 17 03:30:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:28.131Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_packets_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"br_dign_s\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546562882
Jul 17 03:30:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:28.138Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"msh-tsgx10\", asset_id=\"69\", condition=\"unknown\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"coredns-647dbfccd5-sr892\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"ce676e45-7527-495c-945c-c9fa2e1248cd\"}" timestamp=1689546555682
Jul 17 03:30:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:28.234Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"66\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546562882
Jul 17 03:30:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:28.297Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_complete\", asset=\"msh-tsgx10\", asset_id=\"69\", condition=\"unknown\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", job_name=\"helm-install-tsg-traffic-engine-numa-0\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546555682
Jul 17 03:30:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:28.465Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"cortex_kv_request_duration_seconds_bucket\", agent_name=\"msh-kvm-nz001\", application=\"loki\", datacenter=\"MSH\", instance=\"127.0.0.1:13100\", kv_name=\"distributor-lifecycler\", le=\"0.025\", operation=\"CAS\", role=\"primary\", status_code=\"200\", type=\"memberlist\"}" timestamp=1689546568327
Jul 17 03:30:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:28.525Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"loki_chunk_store_series_pre_intersection_per_query_sum\", agent_name=\"msh-kvm-nz001\", application=\"loki\", datacenter=\"MSH\", instance=\"127.0.0.1:13100\"}" timestamp=1689546568327
Jul 17 03:30:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:28.577Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_running\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"firewall\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", pod=\"tsg-traffic-engine-numa-1-firewall-7975457c58-82x8t\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"848571bc-a31e-44de-a0f0-d172cf8f3bd0\"}" timestamp=1689546555682
Jul 17 03:30:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:28.737Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_spec_parallelism\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", job_name=\"helm-install-cm-policy-local-cache\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546555682
Jul 17 03:30:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:28.769Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_created\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-0-firewall-7cbb5bf99c\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546555682
Jul 17 03:30:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:28.929Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_mallocs_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546559446
Jul 17 03:30:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:28.930Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"loki_logql_querystats_bytes_processed_per_seconds_bucket\", agent_name=\"msh-kvm-nz001\", application=\"loki\", datacenter=\"MSH\", instance=\"127.0.0.1:13100\", latency_type=\"fast\", le=\"6e+10\", range=\"instant\", status_code=\"500\", type=\"metric\"}" timestamp=1689546568327
Jul 17 03:30:29 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:29.023Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"47\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546562882
Jul 17 03:30:29 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:29.051Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"57\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546562882
Jul 17 03:30:29 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:29.278Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_state_started\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"coredns\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"coredns-647dbfccd5-sr892\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"ce676e45-7527-495c-945c-c9fa2e1248cd\"}" timestamp=1689546555682
Jul 17 03:30:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:31.026Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_labels\", asset=\"msh-tsgx05\", asset_id=\"67\", configmap=\"shaping-tsg-traffic-engine-numa-1\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546564068
Jul 17 03:30:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:31.105Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_dormant\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth6dcefb50\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546560939
Jul 17 03:30:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:31.163Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db149\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546570141
Jul 17 03:30:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:31.331Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"overlay\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-d4faacec6419ee17bb3b6c4c101758c97b8267ff9322797ba4a5360aa88032d1-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689546561015
Jul 17 03:30:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:31.376Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-cm-policy-local-cache-g54kj\", project=\"TSG-OS\", project_id=\"7\", reason=\"Evicted\", service=\"prometheus-kube-state-metrics\", uid=\"b7366540-fb42-43e9-ab99-2918757ab561\"}" timestamp=1689546564068
Jul 17 03:30:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:31.407Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546560939
Jul 17 03:30:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:31.636Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_mod_info\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"github.com/prometheus/client_model\", nz_agent_id=\"3\", program=\"systemd_exporter\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", version=\"v0.2.0\"}" timestamp=1689546561015
Jul 17 03:30:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:31.656Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_packets_total\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"cadvisor\", interface=\"cnibr0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546553170
Jul 17 03:30:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:31.684Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_link_error_recovery_total\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_1\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546560939
Jul 17 03:30:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:31.718Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"124\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546560939
Jul 17 03:30:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:31.744Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_annotations\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"tsg-traffic-engine-numa-0-firewall\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546570436
Jul 17 03:30:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:31.762Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx03\", asset_id=\"95\", container=\"coredns\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Total\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546555939
Jul 17 03:30:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:31.786Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx05\", asset_id=\"67\", container=\"dashboard-metrics-scraper\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Write\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546550616
Jul 17 03:30:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:31.808Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_free_bytes\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"shm\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/95d5b60888ba291b4d576ad8bf0b3f4ea40c0dc6b50f0b3efcf92f20d93cc451/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546560939
Jul 17 03:30:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:31.824Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_limit_bytes\", asset=\"msh-tsgx05\", asset_id=\"67\", container=\"kube-multus\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546555836
Jul 17 03:30:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:31.879Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"msh-tsgx05\", asset_id=\"67\", condition=\"true\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-cm-policy-local-cache-g54kj\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"b7366540-fb42-43e9-ab99-2918757ab561\"}" timestamp=1689546564068
Jul 17 03:30:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:31.898Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"metrics\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", failure_type=\"pgfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"hierarchy\"}" timestamp=1689546550571
Jul 17 03:30:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:31.908Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"45\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546559093
Jul 17 03:30:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:31.994Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas_unavailable\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"tsg-traffic-engine-numa-0-firewall\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546564068
Jul 17 03:30:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:32.000Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_terminated_reason\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"helm\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-delete-tsg-traffic-engine-numa-1-cxlvk\", project=\"TSG-OS\", project_id=\"7\", reason=\"Completed\", service=\"prometheus-kube-state-metrics\", uid=\"1d75a15d-f414-4a89-93b6-193080fb66a4\"}" timestamp=1689546559276
Jul 17 03:30:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:32.049Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_files_free\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"shm\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/91032bc729b8e1686fa4d94c50384c507e0742062db815fbf5de8383d7a9664c/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546560939
Jul 17 03:30:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:32.065Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_labels\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-pushgateway\", uid=\"ce07cee4-8c10-4905-b16b-1d9bba9dec48\"}" timestamp=1689546559276
Jul 17 03:30:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:32.092Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-systemd-exporter-zbfbx\", project=\"TSG-OS\", project_id=\"7\", reason=\"NodeLost\", service=\"prometheus-kube-state-metrics\", uid=\"417712f3-89ba-43e3-a520-3343986e90fc\"}" timestamp=1689546570436
Jul 17 03:30:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:32.124Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_commands_failed_calls_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cmd=\"config\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546570141
Jul 17 03:30:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:32.135Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_spec_replicas\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-1-firewall-dc9c7db68\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546570436
Jul 17 03:30:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:32.147Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db183\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546570141
Jul 17 03:30:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:32.248Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx05\", asset_id=\"67\", container=\"dashboard-metrics-scraper\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Read\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546550616
Jul 17 03:30:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:32.263Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_ready_replicas\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"cm-policy-local-cache-redis-master-69967b5879\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546564068
Jul 17 03:30:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:32.313Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx05\", asset_id=\"67\", container=\"prometheus-server-configmap-reload\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Discard\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546549675
Jul 17 03:30:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:32.316Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_resident_memory_bytes\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sshd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689546561015
Jul 17 03:30:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:32.341Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"rpcbind.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"target\"}" timestamp=1689546561015
Jul 17 03:30:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:32.359Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_node_status_condition\", asset=\"msh-tsgx05\", asset_id=\"67\", condition=\"MemoryPressure\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", node=\"localhost\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"true\"}" timestamp=1689546564068
Jul 17 03:30:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:32.396Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_restart_policy\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-server-6ff784d758-tq6lp\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", type=\"Always\", uid=\"1a454423-a4c8-40de-896e-6655fed1aa68\"}" timestamp=1689546570436
Jul 17 03:30:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:32.450Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_processes\", asset=\"msh-tsgx05\", asset_id=\"67\", container=\"prometheus-server\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546553428
Jul 17 03:30:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:32.469Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Pending\", pod=\"kube-multus-ds-8krwn\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"da4fef9f-6e38-4557-ac6b-3ab5eb04a34f\"}" timestamp=1689546570436
Jul 17 03:30:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:32.502Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"twa-tsgx03\", asset_id=\"95\", condition=\"false\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-kube-state-metrics-6b768c7bf8-8z25n\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"0f76af51-4cb4-40c5-8ff5-c445f8b7b608\"}" timestamp=1689546570436
Jul 17 03:30:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:32.519Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"114\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546559093
Jul 17 03:30:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:32.557Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"113\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546560939
Jul 17 03:30:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:32.576Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"vethdc7cf888\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546560939
Jul 17 03:30:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:32.619Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db24\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546556711
Jul 17 03:30:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:32.674Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_last_seen\", asset=\"msh-tsgx05\", asset_id=\"67\", container=\"prometheus-alertmanager-configmap-reload\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546555853
Jul 17 03:30:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:32.694Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"19\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546559093
Jul 17 03:30:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:32.713Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_drop_total\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethc8d1fba9\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546559093
Jul 17 03:30:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:32.713Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_cache\", asset=\"msh-tsgx05\", asset_id=\"67\", container=\"kube-multus\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546551799
Jul 17 03:30:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:32.724Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_cache\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689546551000
Jul 17 03:30:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:32.828Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546560791
Jul 17 03:30:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:32.833Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_namespace_status_phase\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-node-lease\", nz_agent_id=\"3\", phase=\"Terminating\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546570436
Jul 17 03:30:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:32.859Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_up\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"cnibr0\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546560791
Jul 17 03:30:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:32.915Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"44\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546560605
Jul 17 03:30:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:32.930Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"58\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546560605
Jul 17 03:30:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:32.947Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_down_changes_total\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"br_dign_s\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546559093
Jul 17 03:30:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:32.959Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas_available\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"prometheus-server\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546564068
Jul 17 03:30:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:33.009Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"process_resident_memory_bytes\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546559093
Jul 17 03:30:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:33.022Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"82\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546560605
Jul 17 03:30:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:33.072Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"11\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546560791
Jul 17 03:30:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:33.084Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"proc-sys-fs-binfmt_misc.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689546562644
Jul 17 03:30:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:33.139Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_reservation_limit_bytes\", asset=\"twa-tsgx05\", asset_id=\"99\", container=\"prometheus-server\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546558535
Jul 17 03:30:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:33.151Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db95\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546569260
Jul 17 03:30:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:33.185Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"14\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546559093
Jul 17 03:30:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:33.198Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_commands_duration_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cmd=\"multi\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546566438
Jul 17 03:30:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:33.222Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_state_started\", asset=\"twa-tsgx05\", asset_id=\"99\", container=\"mrapm-device\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"mrzcpd-exporter-5cf7f967cc-plbbr\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"ac696f24-a067-4de4-82cf-63dca2761ac4\"}" timestamp=1689546559894
Jul 17 03:30:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:33.287Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_virtual_memory_bytes\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sshd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689546567691
Jul 17 03:30:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:33.291Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_dump_payload_sanitizations\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546566438
Jul 17 03:30:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:33.311Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth1ccea153\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546560791
Jul 17 03:30:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:33.322Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_bytes_total\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"eno33np0\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546560791
Jul 17 03:30:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:33.460Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"overlay\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-36bc21e307f94721d2ae3da9e6bac494552d446d3fc34a666e9887b93bfa6d28-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689546562644
Jul 17 03:30:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:33.466Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_virtual_memory_bytes\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"rshim.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689546567691
Jul 17 03:30:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:33.484Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_changes_total\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth01f00b07\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546560791
Jul 17 03:30:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:33.498Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db142\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546556711
Jul 17 03:30:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:33.501Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-61ee0d438537dadde00ed89c72c70e0efd67f6edbdffd8feb1679f29c6191169-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689546562644
Jul 17 03:30:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:33.744Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"126\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546560605
Jul 17 03:30:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:33.818Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_flags\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth0a7ef540\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546560605
Jul 17 03:30:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:33.891Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Discard\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689546543107
Jul 17 03:30:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:33.891Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"113\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546560605
Jul 17 03:30:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:33.908Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_swap_limit_bytes\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689546558535
Jul 17 03:30:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:33.922Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_frame_total\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth17383320\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546560605
Jul 17 03:30:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:33.929Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failcnt\", asset=\"twa-tsgx05\", asset_id=\"99\", container=\"dign-server-dns\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546558138
Jul 17 03:30:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:34.009Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_queue_length\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth989c1124\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546560605
Jul 17 03:30:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:34.027Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_net_dev_group\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens2f1np1\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546560605
Jul 17 03:30:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:34.044Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db208\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546569260
Jul 17 03:30:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:40.480Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:a0-0000:a0:03.1-0000:a1:00.0-net-ens6f0np0.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689546575764
Jul 17 03:30:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:40.749Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"57\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546571647
Jul 17 03:30:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:40.755Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_flushesQueuedCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"firewall_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"08891c45c13dae40a8dde8ce2a2d7dd4\", table=\"index_partfile_firewall_hos_bucket\", type=\"index\"}" timestamp=1689546579756
Jul 17 03:30:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:40.900Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_maxStoreFileAge\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"0853c9106db395a681eba407e5913572\", table=\"session_record_hos_bucket\", type=\"data\"}" timestamp=1689546579756
Jul 17 03:30:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:40.935Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_storeFileSize\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", namespace=\"system\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"b0132e934b40123d0abf0a3c94a74975\", table=\"bucket_meta\", type=\"data\"}" timestamp=1689546579756
Jul 17 03:30:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:40.963Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_10_194_1_metric_deleteBatchTime_25th_percentile\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546579756
Jul 17 03:30:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:41.099Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"118\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546571647
Jul 17 03:30:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:41.179Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"ipmi.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689546575764
Jul 17 03:30:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:41.186Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_labels\", asset=\"twa-tsgx04\", asset_id=\"97\", configmap=\"kubernetes-dashboard-settings\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546580034
Jul 17 03:30:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:41.205Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_restart_policy\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"svclb-traefik-9ef87c92-t5xfr\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", type=\"Always\", uid=\"9024abf3-9329-4f68-b743-c54f18ca214f\"}" timestamp=1689546580034
Jul 17 03:30:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:41.232Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"47\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546571647
Jul 17 03:30:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:41.285Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth4380d8c7\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546571647
Jul 17 03:30:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:41.342Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_flags\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens3f1np1\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546571647
Jul 17 03:30:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:41.360Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"94\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546571647
Jul 17 03:30:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:41.482Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_drop_total\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens8f0np0\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546571647
Jul 17 03:30:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:41.655Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"java_lang_OperatingSystem_TotalPhysicalMemorySize\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546579756
Jul 17 03:30:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:41.678Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_deleteCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"session_record_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"5f7574665e173ad2a674b102d217bdf7\", table=\"index_partfile_session_record_hos_bucket\", type=\"index\"}" timestamp=1689546579756
Jul 17 03:30:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:41.710Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_TASK_metric_deleteTime_median\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546579756
Jul 17 03:30:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:41.766Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_storeCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"firewall_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"bcee559236e0be70ad0a1e9d2a915a48\", table=\"index_partfile_firewall_hos_bucket\", type=\"index\"}" timestamp=1689546579756
Jul 17 03:30:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:41.788Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_10_194_1_metric_compactionTime_98th_percentile\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546579756
Jul 17 03:30:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:41.856Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_vl15_dropped_total\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_3\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546571647
Jul 17 03:30:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:42.176Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.grpc.v1.cri-sandboxes-ad075c5ee04f1613751a7f86c4417e92a42be3cb828f9445243ca3d249d81243-shm.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689546575764
Jul 17 03:30:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:42.231Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_status_active\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", job_name=\"helm-install-tsg-traffic-engine-numa-1\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546580034
Jul 17 03:30:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:42.332Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546571647
Jul 17 03:30:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:42.569Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db214\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546574838
Jul 17 03:30:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:42.654Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_scanCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"assessment_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"6a9f18460a6d21d26922deef74d285c8\", table=\"index_partfile_assessment_hos_bucket\", type=\"index\"}" timestamp=1689546579756
Jul 17 03:30:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:42.659Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_restarts_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", container=\"dign-client\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"dign-client-nmxzm\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"4d40e67b-cb75-462e-b6c7-c667d37b9d55\"}" timestamp=1689546575154
Jul 17 03:30:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:42.723Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_status_active\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", job_name=\"helm-install-traefik\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546575154
Jul 17 03:30:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:42.724Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", cpu=\"54\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546576242
Jul 17 03:30:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:42.740Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db110\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546574838
Jul 17 03:30:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:42.791Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_daemonset_status_number_unavailable\", asset=\"pcap-tsgx01\", asset_id=\"71\", daemonset=\"svclb-traefik-7eb8511e\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546575154
Jul 17 03:30:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:43.075Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_daemonset_status_updated_number_scheduled\", asset=\"pcap-tsgx01\", asset_id=\"71\", daemonset=\"dign-server\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546575154
Jul 17 03:30:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:43.429Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"var-lib-kubelet-pods-a38bc0a0\\\\x2df54d\\\\x2d4548\\\\x2db0f1\\\\x2d6472f0282ca8-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2dqtpg9.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689546580671
Jul 17 03:30:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:43.599Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", container=\"exporter\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"uninterruptible\"}" timestamp=1689546571654
Jul 17 03:30:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:43.618Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"machine_memory_bytes\", asset=\"pcap-tsgx01\", asset_id=\"71\", boot_id=\"0c9f71d0-5bdc-41c7-859b-27d979a8af7b\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"cadvisor\", machine_id=\"1cbd20fa40b34cbeab88c82424422afd\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", system_uuid=\"4c4c4544-005a-3210-8037-b3c04f5a4333\"}" timestamp=1689546580951
Jul 17 03:30:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:43.656Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_scrape_collector_duration_seconds\", asset=\"pcap-tsgx01\", asset_id=\"71\", collector=\"bcache\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546576242
Jul 17 03:30:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:43.701Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_tasks_max\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"NetworkManager-wait-online.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689546580671
Jul 17 03:30:44 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:44.105Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"mysql_global_variables_innodb_disallow_writes\", asset=\"msh-dellr740-olap002\", asset_id=\"13\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Mariadb-msh-dellr740-olap002\", endpoint_id=\"69\", module=\"NC-Mariadb\", module_id=\"43\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546582785
Jul 17 03:30:46 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:46.982Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_TotalCallTime_90th_percentile\", asset=\"pcap-dellr740-dt003\", asset_id=\"35\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt003-Hmaster\", endpoint_id=\"387\", module=\"DC-HBase\", module_id=\"3\", name=\"Master\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"IPC\"}" timestamp=1689546586370
Jul 17 03:30:48 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:48.761Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_appendCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", namespace=\"tsg_galaxy\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"ce0fe72f29202c7873dcf268f8d734d3\", table=\"gtpc_knowledge_base\", type=\"data\"}" timestamp=1689546587439
Jul 17 03:30:48 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:48.832Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_filteredReadRequestCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"knowledge_base_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"d2f78d7225dc730bfcd9893f9f48ed39\", table=\"index_time_knowledge_base_hos_bucket\", type=\"index\"}" timestamp=1689546587439
Jul 17 03:30:48 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:48.903Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_storeCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"default\", project=\"OLAP\", project_id=\"3\", region=\"1ec6c491dc19e54814be0f747b7ceb69\", table=\"index_time_default\", type=\"index\"}" timestamp=1689546587439
Jul 17 03:30:48 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:48.935Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_2_metric_flushMemstoreSize_99th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546587439
Jul 17 03:30:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:49.091Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_194_3_metric_compactionOutputFileCount_99_9th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546587439
Jul 17 03:30:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:49.149Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_rpcMutateRequestCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Server\"}" timestamp=1689546587439
Jul 17 03:30:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:49.333Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_1_metric_majorCompactionInputFileCount_90th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546587439
Jul 17 03:30:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:49.355Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_AppendSize_25th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"WAL\"}" timestamp=1689546587439
Jul 17 03:30:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:49.407Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_194_3_metric_scanSize_num_ops\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546587439
Jul 17 03:30:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:49.596Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_device_id\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth43988c23\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546581797
Jul 17 03:30:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:49.689Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_labels\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"chart-values-mrzcpd-exporter\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546579668
Jul 17 03:30:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:49.757Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_flags\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"eno2\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546581797
Jul 17 03:30:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:49.814Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_net_dev_group\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens4f2np2\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546581797
Jul 17 03:30:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:49.851Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_other_sys_bytes\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546586625
Jul 17 03:30:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:49.979Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_cache\", asset=\"twa-tsgx08\", asset_id=\"105\", container=\"dashboard-metrics-scraper\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546581476
Jul 17 03:30:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:50.053Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"scrape_samples_scraped\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_instance=\"10.42.0.69:8080\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546581568
Jul 17 03:30:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:50.062Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"plymouth-start.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689546586554
Jul 17 03:30:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:50.068Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_queue_length\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethbd8ba9ba\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546581797
Jul 17 03:30:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:50.078Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-6f7082c9\\\\x2d24a1\\\\x2d4f4f\\\\x2daf11\\\\x2dba474eb9a99b-volume\\\\x2dsubpaths-config\\\\x2dvolume-telegraf-1.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689546586554
Jul 17 03:30:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:50.235Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx08\", asset_id=\"105\", container=\"dign-server-dns\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Async\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546573064
Jul 17 03:30:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:50.248Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_constraint_errors_transmitted_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_6\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546586709
Jul 17 03:30:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:50.259Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_node_status_condition\", asset=\"twa-tsgx08\", asset_id=\"105\", condition=\"MemoryPressure\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", node=\"localhost\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"true\"}" timestamp=1689546581568
Jul 17 03:30:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:50.261Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-veth019f8c97.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689546586554
Jul 17 03:30:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:50.269Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", state=\"stopped\"}" timestamp=1689546578744
Jul 17 03:30:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:50.365Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_compressed_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens6f0np0\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546586709
Jul 17 03:30:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:50.388Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_errs_total\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens4f2np2\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546581797
Jul 17 03:30:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:50.422Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_unicast_packets_received_total\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_10\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546581797
Jul 17 03:30:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:50.435Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"twa-tsgx10\", asset_id=\"109\", cpu=\"40\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546581797
Jul 17 03:30:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:50.456Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-infiniband-umad10.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689546581057
Jul 17 03:30:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:50.668Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"16\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546586709
Jul 17 03:30:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:50.685Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_address_not_ready\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_endpoint=\"shaping-announce-port-tsg-traffic-engine-numa-0\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546579668
Jul 17 03:30:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:50.782Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_link_downed_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_10\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546586709
Jul 17 03:30:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:50.859Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db41\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546587556
Jul 17 03:30:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:51.022Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Udp_Close_byT\", app_name=\"sapp4\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", vsysId=\"1\"}" timestamp=1689546578548
Jul 17 03:30:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:51.033Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546586709
Jul 17 03:30:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:51.067Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", cpu=\"14\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-olap003\", endpoint_id=\"235\", mode=\"user\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"yarn_resourcemanager,zookeeper,nacos,kafka,spark,druid_data,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546589714
Jul 17 03:30:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:51.121Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", cpu=\"18\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-olap003\", endpoint_id=\"235\", mode=\"idle\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"yarn_resourcemanager,zookeeper,nacos,kafka,spark,druid_data,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546589714
Jul 17 03:30:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:51.284Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546586709
Jul 17 03:30:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:51.367Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_scrape_collector_duration_seconds\", asset=\"twa-tsgx08\", asset_id=\"105\", collector=\"nfs\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546586709
Jul 17 03:30:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:51.376Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_fifo_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"bond0\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546586709
Jul 17 03:30:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:51.550Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_owner\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", owner_is_controller=\"true\", owner_kind=\"Deployment\", owner_name=\"systemd-exporter\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"systemd-exporter-7f549d9485\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546581568
Jul 17 03:30:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:51.571Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"22\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546586709
Jul 17 03:30:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:51.598Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"93\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546586709
Jul 17 03:30:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:51.625Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"promhttp_metric_handler_requests_total\", asset=\"twa-tsgx08\", asset_id=\"105\", code=\"200\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"ipmi-exporter\"}" timestamp=1689546579627
Jul 17 03:30:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:52.727Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", container=\"exporter\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Async\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546578911
Jul 17 03:30:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:52.762Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_packets_dropped_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"cadvisor\", interface=\"bond0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546588730
Jul 17 03:30:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:52.793Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2dpath-pci\\\\x2d0000:81:00.0\\\\x2dscsi\\\\x2d0:2:0:0\\\\x2dpart5.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689546585496
Jul 17 03:30:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:52.967Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_multicast_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"cnibr0\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546589337
Jul 17 03:30:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:53.031Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"tmpfs\", name=\"data-var-lib-kubelet-pods-9eafc211\\\\x2d7f7b\\\\x2d4e70\\\\x2db713\\\\x2dce422850dd16-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2dwv7sd.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689546585496
Jul 17 03:30:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:53.052Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-f0261fd939d5ee4741dfcc1004f5d46225a1b1ff3f1a0fbdebb8604ad1cb5332-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689546585496
Jul 17 03:30:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:53.244Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_node_status_condition\", asset=\"pcap-tsgx11\", asset_id=\"115\", condition=\"MemoryPressure\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", node=\"localhost\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"unknown\"}" timestamp=1689546584478
Jul 17 03:30:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:53.294Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-ens1f0np0.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689546585496
Jul 17 03:30:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:53.310Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_threads\", asset=\"msh-tsgx12\", asset_id=\"113\", container=\"prometheus-alertmanager\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546575106
Jul 17 03:30:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:53.326Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx12\", asset_id=\"113\", cpu=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546583391
Jul 17 03:30:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:53.334Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"plymouth-switch-root.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689546591179
Jul 17 03:30:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:53.348Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", container=\"dign-server-dns\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Discard\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546578121
Jul 17 03:30:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:53.389Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_drop_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens7f1np1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546589337
Jul 17 03:30:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:53.393Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_reservation_limit_bytes\", asset=\"msh-tsgx12\", asset_id=\"113\", container=\"dign-server-web\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546580589
Jul 17 03:30:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:53.419Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_hwmon_temp_celsius\", asset=\"pcap-dellr740-dt002\", asset_id=\"33\", chip=\"platform_coretemp_1\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"NC-Gohangout-pcap-dellr740-dt002\", endpoint_id=\"173\", module=\"DC-Gohangout\", module_id=\"35\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sensor=\"temp8\"}" timestamp=1689546592666
Jul 17 03:30:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:53.436Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db2\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546579367
Jul 17 03:30:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:53.466Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_ips\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", ip=\"10.42.0.35\", ip_family=\"4\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", pod=\"dashboard-metrics-scraper-5b94897dc9-vm626\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"db0f9f5a-e16e-4bbf-ba42-94fb5a5a260d\"}" timestamp=1689546588663
Jul 17 03:30:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:53.474Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", container=\"configmap-reloader\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Async\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546584818
Jul 17 03:30:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:53.481Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"msh-tsgx12\", asset_id=\"113\", condition=\"false\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"dign-server-ml4qh\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"2ed77d82-be58-47ef-9e9a-55c3cc7c3703\"}" timestamp=1689546588663
Jul 17 03:30:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:53.500Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx08\", asset_id=\"65\", cpu=\"38\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546588328
Jul 17 03:30:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:53.511Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"phy_tx_bits_total\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", dev=\"virtio_dign_s\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"mrzcpd-exporter\"}" timestamp=1689546584559
Jul 17 03:30:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:53.538Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db174\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546578659
Jul 17 03:30:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:53.554Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"pcap-dellr740-dt002\", asset_id=\"33\", cpu=\"6\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"NC-Gohangout-pcap-dellr740-dt002\", endpoint_id=\"173\", module=\"DC-Gohangout\", module_id=\"35\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546592666
Jul 17 03:30:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:53.559Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", container=\"lb-tcp-443\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"stopped\"}" timestamp=1689546579916
Jul 17 03:30:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:53.575Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_xfs_block_map_btree_lookups_total\", asset=\"pcap-dellr740-dt002\", asset_id=\"33\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"sdc1\", endpoint=\"NC-Gohangout-pcap-dellr740-dt002\", endpoint_id=\"173\", module=\"DC-Gohangout\", module_id=\"35\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546592666
Jul 17 03:30:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:53.709Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"msh-tsgx12\", asset_id=\"113\", container=\"metrics-server\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"running\"}" timestamp=1689546570843
Jul 17 03:30:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:53.728Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"106\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546583391
Jul 17 03:30:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:53.755Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx12\", asset_id=\"113\", cpu=\"72\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546583391
Jul 17 03:30:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:53.771Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cpu=\"88\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546579409
Jul 17 03:30:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:53.779Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx12\", asset_id=\"113\", cpu=\"89\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546583391
Jul 17 03:30:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:53.786Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_drop_total\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth068249f9\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546583391
Jul 17 03:30:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:53.805Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cpu=\"44\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546579409
Jul 17 03:30:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:53.814Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_threads\", asset=\"msh-tsgx12\", asset_id=\"113\", container=\"firewall\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689546573315
Jul 17 03:30:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:53.823Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_colls_total\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens2f1np1\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546583391
Jul 17 03:30:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:53.841Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"nsfs\", name=\"run-netns-cni\\\\x2da7a1e1f2\\\\x2d99e0\\\\x2d8dd9\\\\x2d1bf7\\\\x2d2cd30fe0ecdf.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689546580884
Jul 17 03:30:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:53.845Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_sockstat_FRAG_inuse\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546583391
Jul 17 03:30:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:53.849Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_mallocs_total\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546583391
Jul 17 03:30:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:53.871Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_ready_replicas\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"prometheus-pushgateway-bd476bcf9\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546588663
Jul 17 03:30:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:53.940Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-b049c496\\\\x2db9de\\\\x2d40a8\\\\x2d8a1b\\\\x2d4a23997ed556-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2dp4nvf.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689546591179
Jul 17 03:30:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:54.098Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_annotations\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"systemd-exporter-7f549d9485-v7q4v\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"b049c496-b9de-40a8-8a1b-4a23997ed556\"}" timestamp=1689546588663
Jul 17 03:30:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:54.130Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_compressed_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"virtio_dign_s\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546579409
Jul 17 03:30:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:54.260Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"java_lang_MemoryPool_CollectionUsageThreshold\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-Yarn-msh-dellr740-dt001-NodeManager\", endpoint_id=\"351\", module=\"DC-Yarn\", module_id=\"1\", name=\"PS Old Gen\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546593514
Jul 17 03:30:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:54.288Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-veth35760bab.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689546591179
Jul 17 03:30:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:54.316Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx12\", asset_id=\"113\", cpu=\"78\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546583391
Jul 17 03:30:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:54.329Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx12\", asset_id=\"113\", cpu=\"96\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546583391
Jul 17 03:30:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:54.372Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_data_received_bytes_total\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_2\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546588328
Jul 17 03:30:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:54.440Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db112\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546590108
Jul 17 03:30:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:54.503Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Running\", pod=\"helm-install-mrzcpd-exporter-k42ln\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"bd8bd8a8-77b3-49ca-b706-0594b012c60c\"}" timestamp=1689546582058
Jul 17 03:30:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:54.519Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db231\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546578659
Jul 17 03:30:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:54.643Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_state_started\", asset=\"pcap-tsgx07\", asset_id=\"85\", container=\"monit-stream\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"mrzcpd-exporter-5cf7f967cc-8tln6\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"059a389b-c1c6-4cf1-b00e-01a8764f70d8\"}" timestamp=1689546582058
Jul 17 03:30:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:54.664Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cpu=\"11\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546579409
Jul 17 03:30:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:54.710Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_memory_SwapFree_bytes\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546579409
Jul 17 03:30:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:54.718Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_rss\", asset=\"msh-tsgx08\", asset_id=\"65\", container=\"kube-multus\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546555076
Jul 17 03:30:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:54.743Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_packets_dropped_total\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"cadvisor\", interface=\"ens2f1np1\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546576381
Jul 17 03:30:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:54.769Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2dpartuuid-0c4a5088\\\\x2d65da\\\\x2d48ab\\\\x2daa53\\\\x2dfca86eb5eb70.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689546591179
Jul 17 03:30:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:54.801Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-initctl.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"service\"}" timestamp=1689546580884
Jul 17 03:30:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:54.867Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Eth_Bit\", app_name=\"sapp4\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", vsysId=\"1\"}" timestamp=1689546580470
Jul 17 03:30:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:54.942Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"pcap-tsgx07\", asset_id=\"85\", condition=\"false\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-kube-state-metrics-6b768c7bf8-65zmm\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"9942f168-83a7-42b2-bfd0-59b9bd5a3f4b\"}" timestamp=1689546582058
Jul 17 03:30:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:55.096Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_created\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"ipmi-exporter-6f8464dd94\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546582058
Jul 17 03:30:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:55.140Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"msh-tsgx12\", asset_id=\"113\", condition=\"false\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-alertmanager-6df7f457b6-gvtrj\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"67eafe01-721d-41b1-93f6-903b88527edb\"}" timestamp=1689546588663
Jul 17 03:30:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:55.157Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_errs_total\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens4f2np2\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546583391
Jul 17 03:30:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:55.350Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_scrape_collector_duration_seconds\", asset=\"pcap-tsgx07\", asset_id=\"85\", collector=\"btrfs\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546579409
Jul 17 03:30:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:55.354Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"88\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546579409
Jul 17 03:30:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:55.569Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_swap_limit_bytes\", asset=\"pcap-tsgx07\", asset_id=\"85\", container=\"dign-server-dns\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546590608
Jul 17 03:30:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:55.650Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cpu=\"22\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546579409
Jul 17 03:30:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:55.666Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_scrape_collector_duration_seconds\", asset=\"pcap-tsgx07\", asset_id=\"85\", collector=\"edac\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546579409
Jul 17 03:30:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:56.740Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546585883
Jul 17 03:30:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:56.800Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_spec_replicas\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"cm-policy-local-cache-redis-master-69967b5879\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546593389
Jul 17 03:30:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:56.860Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sshd-keygen@ecdsa.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"oneshot\", type=\"service\"}" timestamp=1689546583372
Jul 17 03:30:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:56.909Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas_unavailable\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"local-path-provisioner\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546593389
Jul 17 03:30:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:56.920Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"65\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546585883
Jul 17 03:30:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:56.924Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_labels\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"kubernetes-dashboard-66dd5b456\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546593389
Jul 17 03:30:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:57.040Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546585883
Jul 17 03:30:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:57.249Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"1\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546585883
Jul 17 03:30:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:57.284Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"36\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546585883
Jul 17 03:30:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:57.325Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"debugfs\", name=\"sys-kernel-debug.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689546583372
Jul 17 03:30:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:57.557Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"54\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546585883
Jul 17 03:30:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:30:57.658Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_address_assign_type\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens4f2np2\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546585883
Jul 17 03:31:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:00.390Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_owner\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", owner_is_controller=\"true\", owner_kind=\"ReplicaSet\", owner_name=\"mrzcpd-exporter-5cf7f967cc\", pod=\"mrzcpd-exporter-5cf7f967cc-qcxqj\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"cfc14401-c454-4aca-ae14-7e249e5a7dc0\"}" timestamp=1689546587425
Jul 17 03:31:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:00.408Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_speed_bytes\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens3f1np1\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546589083
Jul 17 03:31:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:00.505Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx01\", asset_id=\"51\", container=\"kubernetes-dashboard\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Sync\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546585405
Jul 17 03:31:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:01.050Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_writes_total\", asset=\"msh-tsgx01\", asset_id=\"51\", container=\"prometheus-pushgateway\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546585026
Jul 17 03:31:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:01.076Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_fifo_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth0943ee2a\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546589083
Jul 17 03:31:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:01.106Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"17\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546589083
Jul 17 03:31:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:01.106Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_restart_policy\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"kube-multus-ds-49nfs\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", type=\"Always\", uid=\"ceaa53db-f09f-4f7d-bfb2-c311be5963c7\"}" timestamp=1689546587425
Jul 17 03:31:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:01.162Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"48\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546589083
Jul 17 03:31:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:01.194Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_physical_state_id\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_7\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546589083
Jul 17 03:31:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:01.221Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"19\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546589083
Jul 17 03:31:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:01.241Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"9\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546589083
Jul 17 03:31:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:01.263Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"115\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546589083
Jul 17 03:31:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:01.401Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-vethf82ad8f5.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689546584879
Jul 17 03:31:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:01.465Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_up\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"eno34np1\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546589083
Jul 17 03:31:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:02.519Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_vmstat_pgmajfault\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546598318
Jul 17 03:31:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:02.666Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"initrd-parse-etc.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"service\"}" timestamp=1689546588447
Jul 17 03:31:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:02.705Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-2dcd40ff3c49e5239d6c0a754ea29ec3dd473a17b8f64bca1554b524d9c65177-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689546588447
Jul 17 03:31:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:02.873Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"lvm2-monitor.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"service\"}" timestamp=1689546588447
Jul 17 03:31:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:02.931Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_restarts_total\", asset=\"msh-tsgx02\", asset_id=\"63\", container=\"prometheus-alertmanager-configmap-reload\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-alertmanager-6df7f457b6-tk42l\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"b4a34040-e633-4749-bfb3-b03089d307fe\"}" timestamp=1689546591412
Jul 17 03:31:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:02.949Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_owner\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", job_name=\"helm-install-prometheus\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", owner_is_controller=\"<none>\", owner_kind=\"<none>\", owner_name=\"<none>\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546591412
Jul 17 03:31:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:02.962Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"104\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546598318
Jul 17 03:31:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:03.291Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", cpu=\"34\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-dt001\", endpoint_id=\"225\", mode=\"steal\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546602988
Jul 17 03:31:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:03.460Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_scrape_collector_duration_seconds\", asset=\"pcap-dellr740-dt002\", asset_id=\"33\", collector=\"filesystem\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"OLAP-Node-Exporter-pcap-dellr740-dt002\", endpoint_id=\"253\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546602497
Jul 17 03:31:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:03.478Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_last_gc_time_seconds\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", instance=\"10.10.20.159:10090\", type=\"global\"}" timestamp=1689546602990
Jul 17 03:31:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:03.488Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"net_conntrack_dialer_conn_failed_total\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", dialer_name=\"173\", instance=\"10.10.20.159:10090\", reason=\"resolution\", type=\"global\"}" timestamp=1689546602990
Jul 17 03:31:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:03.503Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"prometheus_target_sync_failed_total\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", instance=\"10.10.20.159:10090\", scrape_job=\"185\", type=\"global\"}" timestamp=1689546602990
Jul 17 03:31:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:03.641Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Pending\", pod=\"kube-multus-ds-45cnz\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"7adc00b9-e646-4822-9ef0-1a15d34bda7e\"}" timestamp=1689546591412
Jul 17 03:31:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:03.647Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_transmit_wait_total\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_5\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546598318
Jul 17 03:31:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:03.665Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx02\", asset_id=\"63\", cpu=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546598318
Jul 17 03:31:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:03.683Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"packagekit.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"dbus\", type=\"service\"}" timestamp=1689546588447
Jul 17 03:31:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:03.708Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"msh-tsgx02\", asset_id=\"63\", cpu=\"38\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546598318
Jul 17 03:31:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:03.709Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:c0-0000:c0:01.1-0000:c1:00.1-net-ens7f1np1.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689546588447
Jul 17 03:31:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:03.730Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db22\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546589269
Jul 17 03:31:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:03.827Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"nginx_vts_server_requests_total\", asset=\"msh-dellr740-olap002\", asset_id=\"13\", code=\"2xx\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"Galaxy-gateway-nginx-msh-dellr740-olap002\", endpoint_id=\"45\", host=\"192.168.20.222\", module=\"Galaxy-gateway-nginx\", module_id=\"11\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546602934
Jul 17 03:31:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:04.456Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-dellr740-olap005\", asset_id=\"19\", cpu=\"32\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Spark-msh-dellr740-olap005\", endpoint_id=\"17\", mode=\"nice\", module=\"NC-Spark\", module_id=\"47\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546603391
Jul 17 03:31:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:04.680Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"prometheus_target_sync_length_seconds\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", instance=\"10.10.20.159:10090\", quantile=\"0.99\", scrape_job=\"431\", type=\"global\"}" timestamp=1689546602990
Jul 17 03:31:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:06.472Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_sockstat_UDP_mem_bytes\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Spark-msh-dellr740-olap003\", endpoint_id=\"13\", module=\"NC-Spark\", module_id=\"47\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546606091
Jul 17 03:31:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:06.893Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_created\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"localhost.node-password.k3s\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546601227
Jul 17 03:31:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:06.928Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_created\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", uid=\"d6714ab1-120a-4250-b8f5-d757c18f426c\"}" timestamp=1689546601227
Jul 17 03:31:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:07.292Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_annotations\", asset=\"twa-tsgx01\", asset_id=\"91\", configmap=\"cm-cache-extra-scripts\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546601227
Jul 17 03:31:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:07.316Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cpu=\"118\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546603910
Jul 17 03:31:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:07.353Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_max_fds\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"auditd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689546593582
Jul 17 03:31:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:07.379Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db152\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546598290
Jul 17 03:31:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:07.404Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db147\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546598290
Jul 17 03:31:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:07.415Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_spec_replicas\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"metrics-server\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546601227
Jul 17 03:31:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:07.459Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"nsfs\", name=\"run-netns-cni\\\\x2d2a2d6c94\\\\x2d60c2\\\\x2d23db\\\\x2d4ddb\\\\x2d5b6355c3ac29.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689546593582
Jul 17 03:31:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:07.565Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_status_completion_time\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", job_name=\"helm-install-traefik\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546601227
Jul 17 03:31:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:07.841Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sysstat.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"oneshot\", type=\"service\"}" timestamp=1689546593582
Jul 17 03:31:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:08.148Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_info\", asset=\"twa-tsgx01\", asset_id=\"91\", created_by_kind=\"Job\", created_by_name=\"helm-install-tsg-traffic-engine-numa-0\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", host_ip=\"10.10.30.161\", host_network=\"false\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", node=\"localhost\", nz_agent_id=\"3\", pod=\"helm-install-tsg-traffic-engine-numa-0-8qr7q\", pod_ip=\"10.42.0.184\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"fd6761e6-1e9e-4776-b037-dcda2dfd3380\"}" timestamp=1689546601227
Jul 17 03:31:09 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:09.028Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_address_not_ready\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_endpoint=\"shaping-announce-port-tsg-traffic-engine-numa-0\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546601227
Jul 17 03:31:09 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:09.069Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:c0-0000:c0:01.1-0000:c1:00.1-net-ens7f1np1.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689546593582
Jul 17 03:31:09 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:09.609Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_putCount\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"assessment_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"8795d8fa9b6a209d701175f28e5b7624\", table=\"index_filename_assessment_hos_bucket\", type=\"index\"}" timestamp=1689546608807
Jul 17 03:31:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:11.142Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_replicaid\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"default\", project=\"OLAP\", project_id=\"3\", region=\"61ecde15b0197c24043c6c1d3fa7957d\", table=\"index_time_default\", type=\"index\"}" timestamp=1689546608807
Jul 17 03:31:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:11.591Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_numFilesCompactedCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"e62b91d4f4dda56259135f42dc231490\", table=\"firewall_hos_bucket\", type=\"data\"}" timestamp=1689546610147
Jul 17 03:31:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:11.610Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_putCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", namespace=\"SYSTEM\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"995d71939730c37d84c4fc3ea8cfaf7b\", table=\"MUTEX\", type=\"data\"}" timestamp=1689546610147
Jul 17 03:31:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:11.611Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_storeFileSize\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"session_record_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"64ac4b5864b8b0132c1e942230dfc553\", table=\"index_time_session_record_hos_bucket\", type=\"index\"}" timestamp=1689546610147
Jul 17 03:31:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:11.649Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_193_1_metric_splitSuccessCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546610147
Jul 17 03:31:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:11.659Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_2_metric_deleteBatchTime_75th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546610147
Jul 17 03:31:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:11.698Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_STATS_metric_getTime_median\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546610147
Jul 17 03:31:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:11.727Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_flushesQueuedCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"rtp_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"68f6a23c6442f4c42783919a62cdbc95\", table=\"index_time_rtp_hos_bucket\", type=\"index\"}" timestamp=1689546610147
Jul 17 03:31:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:12.651Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_TASK_metric_getTime_90th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546610147
Jul 17 03:31:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:12.691Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_flushesQueuedCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"knowledge_base_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"04ab4d5780512f01a21447ac053d6154\", table=\"index_filename_knowledge_base_hos_bucket\", type=\"index\"}" timestamp=1689546610147
Jul 17 03:31:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:12.701Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_STATS_metric_flushTime_99_9th_percentile\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546608807
Jul 17 03:31:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:12.777Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_default_metric_compactedInputBytes\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546610147
Jul 17 03:31:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:12.788Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_4_metric_storeCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546610147
Jul 17 03:31:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:12.848Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_numBytesCompactedCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"proxy_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"2097b60eaa4ffea308d53e366e4242fb\", table=\"index_filename_proxy_hos_bucket\", type=\"index\"}" timestamp=1689546610147
Jul 17 03:31:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:12.953Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_194_4_metric_scanTime_min\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546610147
Jul 17 03:31:13 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:13.452Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_session_record_hos_bucket_metric_compactionOutputSize_max\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546610147
Jul 17 03:31:13 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:13.469Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_4_metric_compactionInputSize_mean\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546610147
Jul 17 03:31:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:15.288Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"22\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546601230
Jul 17 03:31:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:16.308Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_restarts_total\", asset=\"twa-tsgx07\", asset_id=\"103\", container=\"telegraf-security\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", pod=\"tsg-traffic-engine-numa-0-firewall-5b47889f4f-8qqtj\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"64082a69-3be2-4b5e-9e3a-c0da16576d9e\"}" timestamp=1689546608076
Jul 17 03:31:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:16.777Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"80\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546601230
Jul 17 03:31:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:16.912Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"18\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546601230
Jul 17 03:31:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:16.946Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"67\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546601230
Jul 17 03:31:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:16.977Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"twa-tsgx07\", asset_id=\"103\", condition=\"unknown\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"dign-server-2l4bb\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"69414658-aafa-4782-9372-26c60a463de5\"}" timestamp=1689546608076
Jul 17 03:31:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:16.995Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"shm\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/fc00805fcb8c3c921b545edf164ebddf3a2d4157844624826f07ae710fca20fe/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546601230
Jul 17 03:31:18 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:18.378Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_labels\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_endpoint=\"mrzcpd-exporter\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546608076
Jul 17 03:31:18 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:18.457Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_up\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethe34782c1\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546601230
Jul 17 03:31:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:19.499Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"clickhouse_block_active_time_sdc\", asset=\"twa-dellr740-dt003\", asset_id=\"41\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"NC-Clickhouse-twa-dellr740-dt003\", endpoint_id=\"91\", module=\"NC-Clickhouse\", module_id=\"29\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546618301
Jul 17 03:31:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:19.659Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_oom_events_total\", asset=\"twa-tsgx06\", asset_id=\"101\", container=\"coredns\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546607525
Jul 17 03:31:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:19.719Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"51\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546617612
Jul 17 03:31:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:19.767Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_created\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.traefik.v1\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546613949
Jul 17 03:31:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:19.771Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"overlay\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-460f5be153f52052cd29535bc0132c4b771599ff123f1c6a334c8908afcd72a4-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689546604356
Jul 17 03:31:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:20.150Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_alloc_bytes\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546611998
Jul 17 03:31:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:20.291Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_address_available\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_endpoint=\"kube-dns\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546613949
Jul 17 03:31:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:20.315Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"87\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546617612
Jul 17 03:31:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:20.321Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas_unavailable\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"cm-policy-local-cache-redis-master\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546613949
Jul 17 03:31:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:20.328Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_frame_total\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethcd66f402\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546617612
Jul 17 03:31:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:20.330Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_compressed_total\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"br_dign_c\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546617612
Jul 17 03:31:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:20.394Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"49\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546617612
Jul 17 03:31:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:20.410Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_os_info\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", id=\"rocky\", id_like=\"rhel centos fedora\", module=\"TSG-OS\", module_id=\"77\", name=\"TSG-OS\", nz_agent_id=\"3\", pretty_name=\"TSG-OS v23.07-rc1-079e964 (TSGXNXR620G40R01P0906)\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", version=\"v23.07-rc1-079e964 (TSGXNXR620G40R01P0906)\", version_id=\"8.8\"}" timestamp=1689546608263
Jul 17 03:31:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:20.443Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-c57c2ae94f0670cd653c2d83323b983e033724b86a420701a236d1543724ab30-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689546604356
Jul 17 03:31:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:20.476Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db173\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546611998
Jul 17 03:31:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:20.515Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_resident_memory_bytes\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"rpcbind.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689546604356
Jul 17 03:31:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:20.635Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"remote-fs.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"target\"}" timestamp=1689546610657
Jul 17 03:31:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:21.521Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-efdb7e93\\\\x2d9e91\\\\x2d4d51\\\\x2db008\\\\x2d336f3a7336fb-volumes-kubernetes.io\\\\x7esecret-cm\\\\x2dcache\\\\x2dextra\\\\x2dsecrets.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689546610657
Jul 17 03:31:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:21.842Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"initrd-switch-root.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689546610657
Jul 17 03:31:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:22.370Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Dup_Udp_Bit\", app_name=\"sapp4\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", vsysId=\"1\"}" timestamp=1689546613544
Jul 17 03:31:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:22.485Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", container=\"monit-stream\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"uninterruptible\"}" timestamp=1689546606970
Jul 17 03:31:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:22.580Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_queue_length\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"vethc4731574\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546608263
Jul 17 03:31:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:22.639Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-89821b0de45b57028cc6b74d0b5fdf61639c6280f50be697b3a20e3ee8e7e2b9.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"scope\"}" timestamp=1689546610657
Jul 17 03:31:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:22.731Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"0\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546608263
Jul 17 03:31:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:22.954Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-dellr740-olap004\", asset_id=\"17\", cpu=\"30\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Spark-msh-dellr740-olap004\", endpoint_id=\"15\", mode=\"user\", module=\"NC-Spark\", module_id=\"47\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546622253
Jul 17 03:31:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:23.613Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_multicast_total\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens3f1np1\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546614790
Jul 17 03:31:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:23.718Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx09\", asset_id=\"55\", cpu=\"53\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546614790
Jul 17 03:31:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:23.750Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_scaling_frequency_hertz\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", cpu=\"27\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"OLAP-Node-Exporter-twa-dellr740-dt001\", endpoint_id=\"257\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546621405
Jul 17 03:31:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:23.818Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_memory_MemAvailable_bytes\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546614790
Jul 17 03:31:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:24.077Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_fifo_total\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens4f0np0\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546614790
Jul 17 03:31:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:24.249Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_created\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-traefik-crd-tjh7p\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"2a496308-3fd5-494e-83f9-537bb34e5898\"}" timestamp=1689546620068
Jul 17 03:31:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:24.256Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth6b39ce02\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546614790
Jul 17 03:31:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:24.263Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_allocator_rss_bytes\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546617019
Jul 17 03:31:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:24.314Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"iscsi-onboot.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"service\"}" timestamp=1689546618416
Jul 17 03:31:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:24.361Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"update.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689546618416
Jul 17 03:31:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:24.449Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_info\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"telegraf-security\", container_id=\"containerd://aef4d1595b2e412394a292b96e866c00a753da572177219f200c514acd503d3c\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", image=\"registry.gdnt-cloud.website/tsg-telegraf:v23.07-rc1-079e964\", image_id=\"sha256:2cebbbb83cda415792ef0a3a045c352ff7e754b37cf4f9b9002be26297e1306f\", image_spec=\"registry.gdnt-cloud.website/tsg-telegraf:v23.07-rc1-079e964\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", pod=\"tsg-traffic-engine-numa-1-firewall-cdbcd8f4f-k98cr\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"a27a1ebc-016c-4998-a3f3-85e865dfcd16\"}" timestamp=1689546611122
Jul 17 03:31:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:24.450Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"clevis-luks-askpass.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"service\"}" timestamp=1689546618416
Jul 17 03:31:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:24.598Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db223\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546621795
Jul 17 03:31:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:24.878Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-bb582bdbac574dcf2ffec3a371ea1114bbd4c5fee13d784b7cb41a802d6e491d.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"scope\"}" timestamp=1689546611102
Jul 17 03:31:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:24.922Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-a27a1ebc\\\\x2d016c\\\\x2d4998\\\\x2da3f3\\\\x2d85e865dfcd16-volume\\\\x2dsubpaths-config\\\\x2dvolume-firewall-1.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689546611102
Jul 17 03:31:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:25.658Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_address_available\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_endpoint=\"dashboard-metrics-scraper\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546611122
Jul 17 03:31:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:26.159Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_node_status_capacity\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", node=\"localhost\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", resource=\"cpu\", service=\"prometheus-kube-state-metrics\", unit=\"core\"}" timestamp=1689546620068
Jul 17 03:31:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:26.275Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"45\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546614790
Jul 17 03:31:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:26.282Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546615810
Jul 17 03:31:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:26.285Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_hwmon_sensor_label\", asset=\"pcap-tsgx12\", asset_id=\"117\", chip=\"pci0000:00_0000:00:19_3\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", label=\"tccd8\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", sensor=\"temp10\", service=\"prometheus-node-exporter\"}" timestamp=1689546620160
Jul 17 03:31:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:26.307Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_device_id\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth1e7f8cb3\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546615810
Jul 17 03:31:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:26.329Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"63\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546615810
Jul 17 03:31:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:26.392Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_next_gc_bytes\", asset=\"pcap-kvm-cm001\", asset_id=\"45\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"CM-Redis-pcap-kvm-cm001\", endpoint_id=\"305\", host=\"192.168.10.157\", module=\"CM-Redis\", module_id=\"65\", nz_agent_id=\"3\", parent_asset=\"pcap-dellr740-pve001\", parent_asset_id=\"43\", project=\"Central-Management\", project_id=\"5\"}" timestamp=1689546626190
Jul 17 03:31:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:26.397Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"rshim.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689546613907
Jul 17 03:31:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:26.592Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_ports\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_endpoint=\"prometheus-server\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", port_name=\"http\", port_number=\"9090\", port_protocol=\"TCP\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546620068
Jul 17 03:31:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:26.645Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_running\", asset=\"msh-tsgx09\", asset_id=\"55\", container=\"helm\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-traefik-crd-tjh7p\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"2a496308-3fd5-494e-83f9-537bb34e5898\"}" timestamp=1689546620068
Jul 17 03:31:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:26.690Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-remount-fs.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689546611102
Jul 17 03:31:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:26.731Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-ens4f1np1.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689546618416
Jul 17 03:31:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:26.890Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-fsck-root.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689546611102
Jul 17 03:31:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:26.903Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_virtual_memory_bytes\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"rdma-ndd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689546613907
Jul 17 03:31:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:26.956Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_packets_received_total\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_7\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546615810
Jul 17 03:31:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:26.977Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_rss\", asset=\"twa-tsgx09\", asset_id=\"107\", container=\"prometheus-node-exporter\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546623965
Jul 17 03:31:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:26.987Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"scrape_duration_seconds\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_instance=\"10.42.0.38:9290\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"ipmi-exporter\"}" timestamp=1689546614325
Jul 17 03:31:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:27.516Z caller=dedupe.go:112 component=remote level=error remote_name=52db6f url=http://10.10.10.159:10090/nz-agent/prometheus/proxy/api/v1/write msg="non-recoverable error" count=1000 exemplarCount=0 err="server returned HTTP status 400 : out of order sample"
Jul 17 03:31:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:27.535Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_persistentvolume_status_phase\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", persistentvolume=\"pvc-ec49a672-8a00-422d-a555-1a6bd7871a65\", phase=\"Released\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546615682
Jul 17 03:31:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:27.552Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_flags\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"eno34np1\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546614790
Jul 17 03:31:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:27.619Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-9f850053a09483dbc97381135c2db2e57faae98a6ad05feaec37052d185184ca-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689546614114
Jul 17 03:31:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:27.768Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dnf-makecache.timer\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"timer\"}" timestamp=1689546614114
Jul 17 03:31:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:27.802Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_dormant\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"eno34np1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546620160
Jul 17 03:31:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:27.820Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-vethefb0a7e8.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689546614114
Jul 17 03:31:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:27.831Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"twa-tsgx09\", asset_id=\"107\", container=\"firewall\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", state=\"running\"}" timestamp=1689546621945
Jul 17 03:31:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:27.852Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_errs_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"vethc31b7dd7\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546620160
Jul 17 03:31:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:27.863Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_info\", asset=\"twa-tsgx09\", asset_id=\"107\", configmap=\"kube-root-ca.crt\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546618744
Jul 17 03:31:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:27.871Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_flags\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"vethfc5d9f9d\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546620160
Jul 17 03:31:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:27.904Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas_available\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"prometheus-pushgateway\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546618744
Jul 17 03:31:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:27.910Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_namespace_status_phase\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", phase=\"Active\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546618744
Jul 17 03:31:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:27.945Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"34\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546615810
Jul 17 03:31:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:27.968Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"9\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546615810
Jul 17 03:31:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:28.136Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-virtual-net-vethdb91c360.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689546611102
Jul 17 03:31:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:28.590Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_type\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"chart-values-ipmi-exporter\", service=\"prometheus-kube-state-metrics\", type=\"Opaque\"}" timestamp=1689546618744
Jul 17 03:31:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:28.738Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_annotations\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"prometheus-pushgateway\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546615682
Jul 17 03:31:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:28.780Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db192\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546619446
Jul 17 03:31:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:28.896Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"slice\"}" timestamp=1689546613907
Jul 17 03:31:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:28.923Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.grpc.v1.cri-sandboxes-5cad53f69ddad41ca5a7bdec32118b2fe0949858e656e45ca19234ce16517d1b-shm.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689546613907
Jul 17 03:31:29 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:29.018Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"clickhouse_os_guest_time_cpu19\", asset=\"twa-dellr740-dt002\", asset_id=\"39\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"NC-Clickhouse-twa-dellr740-dt002\", endpoint_id=\"89\", module=\"NC-Clickhouse\", module_id=\"29\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546624866
Jul 17 03:31:29 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:29.582Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_node_status_condition\", asset=\"msh-tsgx10\", asset_id=\"69\", condition=\"Ready\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", node=\"localhost\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"false\"}" timestamp=1689546615682
Jul 17 03:31:29 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:29.640Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"25\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546622882
Jul 17 03:31:29 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:29.655Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"tmpfs\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/var/lib/kubelet/pods/ce676e45-7527-495c-945c-c9fa2e1248cd/volumes/kubernetes.io~projected/kube-api-access-mhtnk\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546622882
Jul 17 03:31:29 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:29.699Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_init_container_status_waiting\", asset=\"twa-tsgx09\", asset_id=\"107\", container=\"install-multus-binary\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"kube-multus-ds-bvcvj\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"680c59bb-c7ab-427d-b652-0e744860b375\"}" timestamp=1689546618744
Jul 17 03:31:29 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:29.777Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_load_average_10s\", asset=\"twa-tsgx09\", asset_id=\"107\", container=\"local-path-provisioner\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546618187
Jul 17 03:31:29 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:29.827Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_system_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", container=\"mrapm-device\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546607598
Jul 17 03:31:30 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:30.001Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_labels\", asset=\"msh-tsgx10\", asset_id=\"69\", configmap=\"chart-content-configmap-reloader\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546615682
Jul 17 03:31:30 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:30.593Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"21\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546622882
Jul 17 03:31:30 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:30.640Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"12\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546622882
Jul 17 03:31:30 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:30.650Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-podc0c07089_8d67_4a93_9c72_855e72ebd4e3.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"slice\"}" timestamp=1689546614114
Jul 17 03:31:30 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:30.702Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"msh-tsgx10\", asset_id=\"69\", condition=\"true\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-delete-tsg-traffic-engine-numa-0-sdpfj\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"9a19dc53-183a-43f4-97c9-ab25098e2098\"}" timestamp=1689546615682
Jul 17 03:31:30 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:30.731Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-396cb6af\\\\x2d9b76\\\\x2d4a32\\\\x2daad2\\\\x2ddeb6dbe16f5b-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2dghr4v.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689546614114
Jul 17 03:31:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:31.402Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_physical_state_id\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_11\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546620939
Jul 17 03:31:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:31.438Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-acb0a639\\\\x2d453c\\\\x2d4a2c\\\\x2da1c6\\\\x2df36ebe18181d-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2dxw2dg.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689546621015
Jul 17 03:31:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:31.978Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_created\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-alertmanager\", uid=\"7874f863-08a4-480c-b7dc-dee717f0f742\"}" timestamp=1689546624068
Jul 17 03:31:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:32.044Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"108\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546620939
Jul 17 03:31:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:32.062Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_node_status_capacity\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", node=\"localhost\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", resource=\"pods\", service=\"prometheus-kube-state-metrics\", unit=\"integer\"}" timestamp=1689546619276
Jul 17 03:31:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:32.124Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_cache\", asset=\"twa-tsgx03\", asset_id=\"95\", container=\"prometheus-server\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546615410
Jul 17 03:31:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:32.140Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Eth_Pkt\", app_name=\"sapp4\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", vsysId=\"1\"}" timestamp=1689546627477
Jul 17 03:31:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:32.144Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_annotations\", asset=\"msh-tsgx11\", asset_id=\"111\", configmap=\"kube-apiserver-legacy-service-account-token-tracking\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546619276
Jul 17 03:31:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:32.155Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2did-wwn\\\\x2d0x62cea7f0add57d002c2c229b337423e6\\\\x2dpart3.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689546622644
Jul 17 03:31:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:32.196Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"19\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546619093
Jul 17 03:31:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:32.215Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_status_completion_time\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", job_name=\"helm-install-systemd-exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546619276
Jul 17 03:31:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:32.227Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db121\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546616711
Jul 17 03:31:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:32.568Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", phase=\"Running\", pod=\"tsg-traffic-engine-numa-1-firewall-6dd49d747-slxl4\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"bdd10b0d-cc33-4a4f-9622-89ebf267e6c3\"}" timestamp=1689546619894
Jul 17 03:31:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:33.014Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"blk-availability.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"oneshot\", type=\"service\"}" timestamp=1689546627691
Jul 17 03:31:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:33.266Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx05\", asset_id=\"67\", container=\"local-path-provisioner\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Total\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546607765
Jul 17 03:31:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:33.268Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"9\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546620605
Jul 17 03:31:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:33.285Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_load_average_10s\", asset=\"msh-tsgx05\", asset_id=\"67\", container=\"kube-state-metrics\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546605755
Jul 17 03:31:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:33.287Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_usage_seconds_total\", asset=\"msh-tsgx05\", asset_id=\"67\", container=\"redis\", cpu=\"total\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546604342
Jul 17 03:31:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:33.303Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_ready\", asset=\"msh-tsgx05\", asset_id=\"67\", container=\"coredns\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"coredns-696d987597-cgfrx\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"84148aa3-78e1-49bc-9b4c-b596d7dd94b8\"}" timestamp=1689546624068
Jul 17 03:31:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:33.326Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_packets_total\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"eno36np3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546620939
Jul 17 03:31:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:33.329Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_compressed_total\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens1f0np0\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546620791
Jul 17 03:31:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:33.362Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_sockets\", asset=\"msh-tsgx05\", asset_id=\"67\", container=\"configmap-reloader\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546600567
Jul 17 03:31:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:33.362Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_system_seconds_total\", asset=\"msh-tsgx05\", asset_id=\"67\", container=\"traefik\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546610292
Jul 17 03:31:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:33.380Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_address_available\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_endpoint=\"mrzcpd-exporter\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546624068
Jul 17 03:31:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:33.681Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"37\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546620791
Jul 17 03:31:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:33.698Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failcnt\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689546607965
Jul 17 03:31:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:33.701Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_labels\", asset=\"msh-tsgx11\", asset_id=\"111\", configmap=\"cluster-dns\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546619276
Jul 17 03:31:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:33.712Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_metadata_resource_version\", asset=\"msh-tsgx05\", asset_id=\"67\", configmap=\"chart-content-configmap-reloader\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546624068
Jul 17 03:31:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:33.719Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_status_start_time\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", job_name=\"helm-install-traefik\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546624068
Jul 17 03:31:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:33.750Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"57\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546620939
Jul 17 03:31:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:33.786Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"68\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546620939
Jul 17 03:31:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:33.806Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"32\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546620605
Jul 17 03:31:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:33.823Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_up_changes_total\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens4f1np1\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546620605
Jul 17 03:31:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:33.973Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"118\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546620939
Jul 17 03:31:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:34.023Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db129\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546626438
Jul 17 03:31:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:34.139Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"msh-tsgx11\", asset_id=\"111\", condition=\"unknown\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-tsg-traffic-engine-numa-1-zrj5x\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"65784633-9e2f-4db8-9d01-240927a62246\"}" timestamp=1689546619276
Jul 17 03:31:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:34.171Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"94\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546620791
Jul 17 03:31:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:34.200Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"tsg_master_mirror_pkts\", app_name=\"tsg_master\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"traffic-engine\", line_name=\"success\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", table_name=\"flow\", vsysId=\"1\"}" timestamp=1689546622297
Jul 17 03:31:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:34.212Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"31\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546620605
Jul 17 03:31:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:34.230Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_compressed_total\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens8f1np1\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546620791
Jul 17 03:31:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:34.332Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_init_container_status_terminated_reason\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"tsg-dign-init\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"dign-server-xqfxk\", project=\"TSG-OS\", project_id=\"7\", reason=\"Completed\", service=\"prometheus-kube-state-metrics\", uid=\"e2593736-e404-42cd-a92c-c9652757c95a\"}" timestamp=1689546619276
Jul 17 03:31:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:34.359Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Unknown\", pod=\"helm-install-tsg-traffic-engine-numa-0-p9rhf\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"105764ab-9632-4df2-829e-d2392e8ab613\"}" timestamp=1689546619276
Jul 17 03:31:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:34.547Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_status_active\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", job_name=\"helm-install-cm-policy-local-cache\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546619894
Jul 17 03:31:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:34.765Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"42\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546620791
Jul 17 03:31:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:34.783Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_scrape_collector_success\", asset=\"twa-tsgx03\", asset_id=\"95\", collector=\"time\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546619093
Jul 17 03:31:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:34.806Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db253\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546626438
Jul 17 03:31:35 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:35.719Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_files\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"shm\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/e4ba4b14a19c466b5a530124fe01c796f501f2cbf6ffef19efc3407f0bac587d/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546620605
Jul 17 03:31:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:40.447Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_writes_total\", asset=\"twa-tsgx04\", asset_id=\"97\", container=\"dign-client\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546623509
Jul 17 03:31:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:40.620Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546631647
Jul 17 03:31:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:40.797Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_CATALOG_metric_numReferenceFiles\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546639757
Jul 17 03:31:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:40.886Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_numFilesCompactedCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"proxy_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"29196d643831de92c5633f334a8ada62\", table=\"index_partfile_proxy_hos_bucket\", type=\"index\"}" timestamp=1689546639757
Jul 17 03:31:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:41.061Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_incrementCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", namespace=\"dos\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"89022c758cf27cca0a6f1b254f97043b\", table=\"ddos_traffic_baselines\", type=\"data\"}" timestamp=1689546639757
Jul 17 03:31:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:41.084Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_hbase_table_rsgroup_metric_splitTime_min\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546639757
Jul 17 03:31:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:41.594Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx04\", asset_id=\"97\", container=\"prometheus-alertmanager\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Write\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546614335
Jul 17 03:31:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:41.954Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"8\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546631647
Jul 17 03:31:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:42.001Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_device_id\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth4e7244c8\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546631647
Jul 17 03:31:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:42.492Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_load_average_10s\", asset=\"twa-tsgx04\", asset_id=\"97\", container=\"telegraf\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689546617820
Jul 17 03:31:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:42.518Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"119\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546631647
Jul 17 03:31:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:42.618Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"network-pre.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"target\"}" timestamp=1689546640671
Jul 17 03:31:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:42.647Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_info\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_endpoint=\"metrics-server\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546635154
Jul 17 03:31:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:42.856Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_numReferenceFiles\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"proxy_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"104e043c58ad6a4a5f519c65c2e6e973\", table=\"index_filename_proxy_hos_bucket\", type=\"index\"}" timestamp=1689546639757
Jul 17 03:31:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:43.469Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_memory_DirectMap1G_bytes\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546631647
Jul 17 03:31:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:43.499Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_ready_replicas\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"cm-policy-local-cache-redis-master-69967b5879\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546635154
Jul 17 03:31:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:43.535Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_rate_bytes_per_second\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"mlx5_6\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546636242
Jul 17 03:31:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:43.608Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_10_194_1_metric_scanSize_num_ops\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546639757
Jul 17 03:31:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:43.700Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_LOG_metric_tableReadQueryPerSecond_mean_rate\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546639757
Jul 17 03:31:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:43.837Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", cpu=\"18\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546636242
Jul 17 03:31:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:43.959Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-d20d4034\\\\x2d164e\\\\x2d4450\\\\x2da851\\\\x2d056ee84ef97e-volume\\\\x2dsubpaths-config\\\\x2dvolume-telegraf-2.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689546640671
Jul 17 03:31:44 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:44.008Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"k3s.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"service\"}" timestamp=1689546640671
Jul 17 03:31:45 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:45.356Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2did-wwn\\\\x2d0x62cea7f0848255002c2c7c5e823af8ea\\\\x2dpart4.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689546640671
Jul 17 03:31:46 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:46.562Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_memory_Slab_bytes\", asset=\"msh-kvm-cm001\", asset_id=\"3\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"CM-Node-Exporter-msh-kvm-cm001\", endpoint_id=\"391\", module=\"CM-Node-Exporter\", module_id=\"75\", nz_agent_id=\"3\", parent_asset=\"msh-dellr740-pve001\", parent_asset_id=\"1\", project=\"Central-Management\", project_id=\"5\"}" timestamp=1689546644238
Jul 17 03:31:48 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:48.739Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_hbase_table_rsgroup_metric_tableWriteQueryPerSecond_mean_rate\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546647439
Jul 17 03:31:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:49.027Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_1_metric_appendTime_median\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546647439
Jul 17 03:31:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:49.050Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_hbase_table_namespace_metric_splitTime_min\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546647439
Jul 17 03:31:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:49.051Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_LOG_metric_tableReadQueryPerSecond_1min_rate\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546647439
Jul 17 03:31:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:49.067Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_putCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"troubleshooting_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"b70eb299f64649891465978a22b50d75\", table=\"index_partfile_troubleshooting_hos_bucket\", type=\"index\"}" timestamp=1689546647439
Jul 17 03:31:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:49.363Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_2_metric_compactionOutputFileCount_75th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546647439
Jul 17 03:31:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:49.386Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_storeCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"knowledge_base_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"56beb531d3f9e76ac417919ebcd4d1b6\", table=\"index_partfile_knowledge_base_hos_bucket\", type=\"index\"}" timestamp=1689546647439
Jul 17 03:31:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:49.870Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_io_time_weighted_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"overlay_0-53\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546642906
Jul 17 03:31:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:49.906Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_persistentvolumeclaim_info\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", persistentvolumeclaim=\"prometheus-alertmanager\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", storageclass=\"local-path\", volumename=\"pvc-c08dcd6c-f603-4169-ae73-c85cac6a245f\"}" timestamp=1689546641568
Jul 17 03:31:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:49.949Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-e94a9495b59cd1a6e9531262f0bd6dc1693cc39a75c93bbdee486bc070e97f1c-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689546646554
Jul 17 03:31:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:50.000Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_hbase_table_rsgroup_metric_getTime_99th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546647439
Jul 17 03:31:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:50.004Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", pod=\"kubernetes-dashboard-66dd5b456-89kng\", project=\"TSG-OS\", project_id=\"7\", reason=\"Evicted\", service=\"prometheus-kube-state-metrics\", uid=\"69257332-0d52-4d37-98ae-1ff0fd197fb1\"}" timestamp=1689546639668
Jul 17 03:31:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:50.606Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx10\", asset_id=\"109\", cpu=\"12\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546641797
Jul 17 03:31:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:51.310Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"twa-tsgx10\", asset_id=\"109\", cpu=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546641797
Jul 17 03:31:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:51.372Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_expired_keys_total\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546646625
Jul 17 03:31:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:51.587Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_edac_csrow_uncorrectable_errors_total\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", controller=\"3\", csrow=\"unknown\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-olap003\", endpoint_id=\"235\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"yarn_resourcemanager,zookeeper,nacos,kafka,spark,druid_data,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546649713
Jul 17 03:31:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:51.597Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_limit_bytes\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"kube-state-metrics\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546639050
Jul 17 03:31:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:51.598Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_slowlog_last_id\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546646625
Jul 17 03:31:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:51.621Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_created\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"mrzcpd-exporter-5cf7f967cc-b62w2\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"c4ec4f52-224c-4f5b-9432-db80b6600279\"}" timestamp=1689546641568
Jul 17 03:31:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:51.648Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db191\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546647556
Jul 17 03:31:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:51.649Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_errors_received_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_3\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546646709
Jul 17 03:31:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:51.757Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_scaling_frequency_min_hertz\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"106\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546646709
Jul 17 03:31:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:51.796Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_errs_total\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth34c9090\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-olap003\", endpoint_id=\"235\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"yarn_resourcemanager,zookeeper,nacos,kafka,spark,druid_data,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546649713
Jul 17 03:31:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:51.838Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx08\", asset_id=\"105\", container=\"dign-client\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Sync\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546636494
Jul 17 03:31:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:51.885Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_usage_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", container=\"exporter\", cpu=\"total\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546638621
Jul 17 03:31:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:51.915Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_cache\", asset=\"twa-tsgx08\", asset_id=\"105\", container=\"local-path-provisioner\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546645333
Jul 17 03:31:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:51.956Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-pod59e0fcfb_c5b9_4bb0_aefc_a15fed75746e.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"slice\"}" timestamp=1689546646554
Jul 17 03:31:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:52.019Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db194\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546647556
Jul 17 03:31:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:52.067Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_2_metric_compactionOutputSize_25th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546647439
Jul 17 03:31:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:52.345Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_info\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"ipmi-exporter\", version=\"go1.13.10\"}" timestamp=1689546644468
Jul 17 03:31:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:52.403Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_ports\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_endpoint=\"prometheus-alertmanager\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", port_name=\"http\", port_number=\"9093\", port_protocol=\"TCP\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546639668
Jul 17 03:31:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:52.595Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_reservation_limit_bytes\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"monit-stream\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546639050
Jul 17 03:31:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:52.646Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_namespace_status_phase\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", phase=\"Terminating\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546639668
Jul 17 03:31:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:52.690Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_restart_policy\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-node-exporter-bz7wf\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", type=\"Always\", uid=\"d58b8827-5397-48fd-9ce6-3976a10dcb09\"}" timestamp=1689546639668
Jul 17 03:31:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:53.215Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-olap003\", endpoint_id=\"235\", module=\"OLAP-Node-Exporter\", module_id=\"51\", name=\"18\", nz_agent_id=\"3\", olap_node_exporter=\"yarn_resourcemanager,zookeeper,nacos,kafka,spark,druid_data,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\", type=\"Processor\"}" timestamp=1689546649713
Jul 17 03:31:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:53.237Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_labels\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"systemd-exporter\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546648950
Jul 17 03:31:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:53.315Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx12\", asset_id=\"113\", container=\"prometheus-server-configmap-reload\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Discard\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546624144
Jul 17 03:31:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:53.394Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_restarts_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", container=\"helm\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-configmap-reloader-smwwp\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"26f82780-5515-438b-98f1-fdfb1a2c1042\"}" timestamp=1689546644478
Jul 17 03:31:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:53.402Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-netns-cni\\\\x2ded73d293\\\\x2d09f7\\\\x2d79a9\\\\x2d9b90\\\\x2d8357f48838d9.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689546645496
Jul 17 03:31:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:53.411Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_net_dev_group\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens2f1np1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546649337
Jul 17 03:31:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:53.462Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_drop_total\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"eno2\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546643391
Jul 17 03:31:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:53.490Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_changes_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth41c1253b\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546649337
Jul 17 03:31:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:53.535Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", cpu=\"14\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546649337
Jul 17 03:31:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:53.573Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_threads\", asset=\"pcap-tsgx11\", asset_id=\"115\", container=\"telegraf\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689546640478
Jul 17 03:31:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:53.599Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", cpu=\"55\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546649337
Jul 17 03:31:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:53.655Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db182\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546639367
Jul 17 03:31:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:53.762Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_threads_max\", asset=\"pcap-tsgx07\", asset_id=\"85\", container=\"kube-state-metrics\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546641905
Jul 17 03:31:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:53.885Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_scrape_collector_success\", asset=\"pcap-tsgx11\", asset_id=\"115\", collector=\"zfs\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546649337
Jul 17 03:31:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:54.101Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_cpu_shares\", asset=\"pcap-tsgx07\", asset_id=\"85\", container=\"lb-tcp-80\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546650608
Jul 17 03:31:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:54.101Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2did-scsi\\\\x2dSDELL_PERC_H345_Adp_00f611881d94842c2c008cd5adf0a7ce\\\\x2dpart4.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689546645496
Jul 17 03:31:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:54.122Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_hwmon_temp_celsius\", asset=\"pcap-tsgx07\", asset_id=\"85\", chip=\"pci0000:00_0000:00:19_3\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", sensor=\"temp4\", service=\"prometheus-node-exporter\"}" timestamp=1689546639409
Jul 17 03:31:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:54.169Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-34d06245e6306777e62cac2af62177d46a07b3092c0f76399f1329200a6c476d.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"scope\"}" timestamp=1689546651179
Jul 17 03:31:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:54.181Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-b4e1ea9a1bdf46e1c045aa09601605abd5c1154bd72d8029126fbd9ca093e129-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689546640884
Jul 17 03:31:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:54.231Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_hwmon_temp_max_celsius\", asset=\"pcap-dellr740-dt002\", asset_id=\"33\", chip=\"platform_coretemp_1\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"NC-Gohangout-pcap-dellr740-dt002\", endpoint_id=\"173\", module=\"DC-Gohangout\", module_id=\"35\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sensor=\"temp10\"}" timestamp=1689546652666
Jul 17 03:31:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:54.241Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cpu=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546639409
Jul 17 03:31:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:54.253Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_cpu_period\", asset=\"pcap-tsgx11\", asset_id=\"115\", container=\"prometheus-pushgateway\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546650039
Jul 17 03:31:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:54.270Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cpu=\"109\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546639409
Jul 17 03:31:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:54.303Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db135\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546639367
Jul 17 03:31:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:54.306Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"tmpfs\", name=\"data-var-lib-kubelet-pods-83b77ff9\\\\x2d13b6\\\\x2d4ffa\\\\x2d841d\\\\x2da87630d4cd3b-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2d4n55q.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689546652135
Jul 17 03:31:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:54.324Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-cbfd93d74de91da2efaf980c1fead763d212f23c3778fc03fe7fbec5875b94fe-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689546652135
Jul 17 03:31:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:54.338Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_created\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_endpoint=\"prometheus-alertmanager\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546642058
Jul 17 03:31:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:54.378Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_virtual_memory_max_bytes\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"tuned.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689546652135
Jul 17 03:31:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:54.409Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_annotations\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.traefik.v1\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546648950
Jul 17 03:31:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:54.450Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"msh-tsgx08\", asset_id=\"65\", container=\"prometheus-pushgateway\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"cadvisor\", failure_type=\"pgfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"container\"}" timestamp=1689546621010
Jul 17 03:31:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:54.453Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db49\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546650108
Jul 17 03:31:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:54.466Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_mapped_file\", asset=\"msh-tsgx08\", asset_id=\"65\", container=\"metrics\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546629931
Jul 17 03:31:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:54.471Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"loadmodules.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"service\"}" timestamp=1689546652135
Jul 17 03:31:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:54.517Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_oom_events_total\", asset=\"msh-tsgx08\", asset_id=\"65\", container=\"coredns\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546621932
Jul 17 03:31:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:54.538Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_writes_merged_total\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546636777
Jul 17 03:31:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:54.544Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx08\", asset_id=\"65\", container=\"coredns\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Read\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546621932
Jul 17 03:31:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:54.555Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db158\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546650108
Jul 17 03:31:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:54.564Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx08\", asset_id=\"65\", cpu=\"79\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546648328
Jul 17 03:31:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:54.636Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_protocol_type\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"virtio_dign_c\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546648328
Jul 17 03:31:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:54.873Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db247\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546650108
Jul 17 03:31:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:54.955Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx08\", asset_id=\"65\", cpu=\"115\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546648328
Jul 17 03:31:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:54.959Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_device_id\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens6f1np1\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546648328
Jul 17 03:31:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:54.979Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cpupower.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689546652135
Jul 17 03:31:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:55.020Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_compressed_total\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"br_dign_c\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546648328
Jul 17 03:31:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:55.087Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_address_not_ready\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_endpoint=\"prometheus-pushgateway\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546648950
Jul 17 03:31:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:55.108Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx08\", asset_id=\"65\", cpu=\"106\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546648328
Jul 17 03:31:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:55.137Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_annotations\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"prometheus-server-6ff784d758\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546648950
Jul 17 03:31:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:55.167Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_metadata_generation\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"coredns-749bdddcdf\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546648950
Jul 17 03:31:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:55.196Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_memory_Unevictable_bytes\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546648328
Jul 17 03:31:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:55.221Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:20-0000:20:03.1-0000:21:00.1-infiniband-mlx5_3.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689546652135
Jul 17 03:31:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:55.241Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_owner\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", owner_is_controller=\"true\", owner_kind=\"Deployment\", owner_name=\"tsg-traffic-engine-numa-0-firewall\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-0-firewall-7cbb5bf99c\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546648950
Jul 17 03:31:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:55.267Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-rfkill.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689546651179
Jul 17 03:31:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:55.282Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_packets_total\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth081275ce\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546643391
Jul 17 03:31:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:55.285Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"msh-tsgx08\", asset_id=\"65\", cpu=\"96\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546648328
Jul 17 03:31:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:55.304Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"vethba64267c\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546643391
Jul 17 03:31:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:55.307Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"msh-tsgx12\", asset_id=\"113\", cpu=\"37\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546643391
Jul 17 03:31:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:55.320Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_compressed_total\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"vethcd3875c5\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546643391
Jul 17 03:31:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:55.480Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_state_started\", asset=\"msh-tsgx12\", asset_id=\"113\", container=\"prometheus-alertmanager-configmap-reload\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-alertmanager-6df7f457b6-gvtrj\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"67eafe01-721d-41b1-93f6-903b88527edb\"}" timestamp=1689546648663
Jul 17 03:31:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:55.579Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db205\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546638659
Jul 17 03:31:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:56.162Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sysstat-collect.timer\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"timer\"}" timestamp=1689546640884
Jul 17 03:31:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:56.255Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_owner\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", owner_is_controller=\"true\", owner_kind=\"ReplicaSet\", owner_name=\"cm-policy-local-cache-redis-master-69967b5879\", pod=\"cm-policy-local-cache-redis-master-69967b5879-mnc4l\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"ae4f02ae-808a-4f27-b376-d8b3a79b4ccd\"}" timestamp=1689546642058
Jul 17 03:31:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:56.744Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"38\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546645883
Jul 17 03:31:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:56.774Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_defrag_key_misses\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546646475
Jul 17 03:31:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:56.776Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled_time\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"dign-client-qkc6p\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"0acf4d0b-d722-41b5-aee0-e6bd3bc175ed\"}" timestamp=1689546653389
Jul 17 03:31:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:56.779Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db47\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546646475
Jul 17 03:31:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:56.794Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"20\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546645883
Jul 17 03:31:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:56.796Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-50d7ffe1608789ad4eade50ba83cbe9c1a6d28867ad613b65a81b6b82c8a3095.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"scope\"}" timestamp=1689546643372
Jul 17 03:31:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:56.798Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_hwmon_chip_names\", asset=\"pcap-tsgx03\", asset_id=\"73\", chip=\"0000:80:01_2_0000:82:00_3\", chip_name=\"bnxt_en\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546645883
Jul 17 03:31:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:56.800Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_labels\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.prometheus.v1\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546653389
Jul 17 03:31:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:56.822Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_labels\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"kubernetes-dashboard\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546653389
Jul 17 03:31:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:56.839Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"openibd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689546643372
Jul 17 03:31:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:56.839Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"72\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546645883
Jul 17 03:31:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:56.839Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2dlabel-TSG\\\\x2dOS\\\\x2dDATA.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689546643372
Jul 17 03:31:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:56.877Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_state_id\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"mlx5_9\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546645883
Jul 17 03:31:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:56.884Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_observed_generation\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"local-path-provisioner\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546653389
Jul 17 03:31:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:56.888Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"122\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546645883
Jul 17 03:31:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:56.899Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"32\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546645883
Jul 17 03:31:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:56.905Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda5\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", fstype=\"ext4\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/var/lib/kubelet/pods/5b5ea4ec-9b19-421d-88cb-9ffacec87f13/volume-subpaths/config-volume/telegraf/1\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546645883
Jul 17 03:31:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:56.926Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"25\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546645883
Jul 17 03:31:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:57.262Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"33\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546645883
Jul 17 03:31:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:31:59.064Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", cpu=\"22\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"OLAP-Node-Exporter-pcap-dellr740-dt001\", endpoint_id=\"251\", mode=\"nice\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546657557
Jul 17 03:32:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:01.007Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_commands_failed_calls_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cmd=\"exec\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546645071
Jul 17 03:32:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:01.022Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_id\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"br_dign_s\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546649083
Jul 17 03:32:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:01.022Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-c6ffcf61\\\\x2d3641\\\\x2d43ac\\\\x2db76c\\\\x2db24b78ac0cd3-volume\\\\x2dsubpaths-config\\\\x2dpath-dign\\\\x2dserver\\\\x2ddns-1.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689546644879
Jul 17 03:32:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:01.035Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"119\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546649083
Jul 17 03:32:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:01.037Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db13\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546645071
Jul 17 03:32:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:01.110Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"24\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546649083
Jul 17 03:32:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:01.138Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"msh-tsgx01\", asset_id=\"51\", container=\"dign-server-dns\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"cadvisor\", failure_type=\"pgfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"container\"}" timestamp=1689546650083
Jul 17 03:32:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:01.179Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_working_set_bytes\", asset=\"msh-tsgx01\", asset_id=\"51\", container=\"metrics\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546651966
Jul 17 03:32:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:01.200Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_created\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"chart-values-tsg-traffic-engine-numa-1\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546647425
Jul 17 03:32:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:01.872Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"101\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546649083
Jul 17 03:32:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:02.504Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_labels\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"configmap-reloader-6c55bbc49f-nkg6m\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"1dda3932-520e-4976-8890-a1a3ca5c9e50\"}" timestamp=1689546651412
Jul 17 03:32:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:02.673Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx02\", asset_id=\"63\", cpu=\"107\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546658318
Jul 17 03:32:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:02.725Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", phase=\"Unknown\", pod=\"tsg-traffic-engine-numa-1-firewall-76f6cf59ff-9hvcf\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"4a2303d1-fff8-4c08-a906-9813964074f7\"}" timestamp=1689546651412
Jul 17 03:32:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:02.847Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_mapped_file\", asset=\"msh-tsgx02\", asset_id=\"63\", container=\"telegraf\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689546642848
Jul 17 03:32:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:02.916Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-33a0974734b1ad76913b0d5e48e88656943d8ee0ce4d83ee9ea4797185a293c7-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689546648447
Jul 17 03:32:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:02.959Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx02\", asset_id=\"63\", cpu=\"66\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546658318
Jul 17 03:32:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:02.966Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx02\", asset_id=\"63\", container=\"configmap-reloader\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Async\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546641572
Jul 17 03:32:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:02.981Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db220\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546649269
Jul 17 03:32:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:03.037Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"msh-tsgx02\", asset_id=\"63\", container=\"lb-tcp-443\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"running\"}" timestamp=1689546628257
Jul 17 03:32:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:03.055Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx02\", asset_id=\"63\", cpu=\"86\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546658318
Jul 17 03:32:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:03.065Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_daemonset_metadata_generation\", asset=\"msh-tsgx02\", asset_id=\"63\", daemonset=\"svclb-traefik-1a2acd8b\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546651412
Jul 17 03:32:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:03.079Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"ftx_pkts_total\", app=\"sapp4-0\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"nf_0_fw\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"mrzcpd-exporter\", serviceFunctionIndex=\"0\"}" timestamp=1689546657995
Jul 17 03:32:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:03.081Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx02\", asset_id=\"63\", cpu=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546658318
Jul 17 03:32:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:03.101Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_name_assign_type\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"eno33np0\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546658318
Jul 17 03:32:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:03.125Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_sockstat_RAW6_inuse\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546658318
Jul 17 03:32:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:03.132Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx02\", asset_id=\"63\", container=\"mrapm-device\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Sync\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546650032
Jul 17 03:32:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:03.140Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_info\", asset=\"msh-tsgx02\", asset_id=\"63\", container=\"helm\", container_id=\"containerd://7434a3d041286d5066968600fcbd4e252a8ace80e2eaedd2be32f383a47a14bb\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", image=\"docker.io/rancher/klipper-helm:v0.8.0-build20230510\", image_id=\"sha256:6f42df210d7faa0b6eb56f5f28bb061930c7834af1e7dd4b3586108ec6b1485b\", image_spec=\"rancher/klipper-helm:v0.8.0-build20230510\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-configmap-reloader-5tp4b\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"26219aeb-0511-4104-9986-0d95f0ea0019\"}" timestamp=1689546651412
Jul 17 03:32:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:03.145Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_annotations\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_endpoint=\"prometheus-alertmanager\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546651412
Jul 17 03:32:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:03.163Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"msh-tsgx02\", asset_id=\"63\", cpu=\"100\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546658318
Jul 17 03:32:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:03.181Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"scrape_samples_scraped\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_instance=\"10.10.20.162:9100\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546658318
Jul 17 03:32:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:03.190Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx02\", asset_id=\"63\", cpu=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546658318
Jul 17 03:32:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:03.241Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_annotations\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"traefik-64f55bb67d-lpmmw\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"11efef4a-05e0-44ee-8681-0e306eccb51c\"}" timestamp=1689546651412
Jul 17 03:32:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:03.247Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_module_fork_in_progress\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546649269
Jul 17 03:32:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:03.251Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_annotations\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"ipmi-exporter-6f8464dd94\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546651412
Jul 17 03:32:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:03.268Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_max_usage_bytes\", asset=\"msh-tsgx02\", asset_id=\"63\", container=\"local-path-provisioner\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546649946
Jul 17 03:32:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:03.284Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx02\", asset_id=\"63\", cpu=\"4\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546658318
Jul 17 03:32:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:03.383Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"msh-tsgx02\", asset_id=\"63\", cpu=\"25\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546658318
Jul 17 03:32:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:03.584Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_timex_estimated_error_seconds\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-dt001\", endpoint_id=\"225\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546662988
Jul 17 03:32:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:03.604Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_frees_total\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", instance=\"10.10.20.159:10090\", type=\"global\"}" timestamp=1689546662989
Jul 17 03:32:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:04.603Z caller=dedupe.go:112 component=remote level=info remote_name=52db6f url=http://10.10.10.159:10090/nz-agent/prometheus/proxy/api/v1/write msg="Dropped sample for series that was not explicitly dropped via relabelling" ref=4436650
Jul 17 03:32:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:07.291Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"78\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546663910
Jul 17 03:32:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:07.309Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_mtu_bytes\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens6f0np0\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546663910
Jul 17 03:32:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:07.440Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_reservation_limit_bytes\", asset=\"twa-tsgx01\", asset_id=\"91\", container=\"kube-multus\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546662966
Jul 17 03:32:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:07.458Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"twa-tsgx01\", asset_id=\"91\", condition=\"Available\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"configmap-reloader\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"true\"}" timestamp=1689546661227
Jul 17 03:32:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:07.550Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dracut-mount.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"service\"}" timestamp=1689546653582
Jul 17 03:32:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:07.573Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cpu=\"30\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546663910
Jul 17 03:32:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:07.592Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_name_assign_type\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens4f3np3\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546663910
Jul 17 03:32:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:07.597Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_mtu_bytes\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens1f0np0\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546663910
Jul 17 03:32:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:07.625Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"blk-availability.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"oneshot\", type=\"service\"}" timestamp=1689546653582
Jul 17 03:32:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:07.650Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_created\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"traefik\", uid=\"2d88d210-fe72-4f50-99f4-eb00173d2876\"}" timestamp=1689546661227
Jul 17 03:32:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:07.664Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_daemonset_labels\", asset=\"twa-tsgx01\", asset_id=\"91\", daemonset=\"dign-client\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546661227
Jul 17 03:32:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:07.765Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"75\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546663910
Jul 17 03:32:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:07.820Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_spec_replicas\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"kubernetes-dashboard\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546661227
Jul 17 03:32:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:07.993Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"99\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546663910
Jul 17 03:32:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:08.185Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_restart_policy\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"mrzcpd-exporter-5cf7f967cc-97flx\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", type=\"Always\", uid=\"083e9092-7400-4cad-afc0-cffbf05464a4\"}" timestamp=1689546661227
Jul 17 03:32:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:08.191Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_write_seconds_total\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda2\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546653596
Jul 17 03:32:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:08.209Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cpu=\"23\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546663910
Jul 17 03:32:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:08.314Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"cortex_ingester_chunk_utilization_bucket\", agent_name=\"msh-kvm-nz001\", application=\"cortex\", datacenter=\"MSH\", instance=\"127.0.0.1:10093\", le=\"0.0\"}" timestamp=1689546667266
Jul 17 03:32:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:08.325Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-veth35124140.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689546653582
Jul 17 03:32:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:08.363Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"metrics-server-54dc485875-l4kw8\", project=\"TSG-OS\", project_id=\"7\", reason=\"NodeAffinity\", service=\"prometheus-kube-state-metrics\", uid=\"e96787f3-8c20-4c5e-a395-ec874ca09072\"}" timestamp=1689546661227
Jul 17 03:32:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:08.394Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_created\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-tsg-traffic-engine-numa-0-8qr7q\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"fd6761e6-1e9e-4776-b037-dcda2dfd3380\"}" timestamp=1689546661227
Jul 17 03:32:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:08.412Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"twa-tsgx01\", asset_id=\"91\", condition=\"unknown\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"traefik-64f55bb67d-bd6jb\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"377e59e7-1c6e-433a-a40c-92bbd7c96064\"}" timestamp=1689546661227
Jul 17 03:32:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:08.488Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_completion_time\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-cm-policy-local-cache-x2s95\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"a269b532-737c-464a-963c-78a3bfa6b3d6\"}" timestamp=1689546661227
Jul 17 03:32:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:08.513Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cpu=\"124\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546663910
Jul 17 03:32:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:08.537Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"twa-tsgx01\", asset_id=\"91\", condition=\"false\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", pod=\"tsg-traffic-engine-numa-1-firewall-7f6cb768db-2lm6c\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"9b7a7ee8-3d4c-4d13-b2bb-d0637b89d0c8\"}" timestamp=1689546661227
Jul 17 03:32:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:08.555Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"chronyd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"forking\", type=\"service\"}" timestamp=1689546653582
Jul 17 03:32:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:08.622Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_labels\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.traefik-crd.v1\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546661227
Jul 17 03:32:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:08.642Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_queue_length\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"br_dign_c\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546663910
Jul 17 03:32:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:08.679Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_gc_duration_seconds\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", quantile=\"0\", service=\"ipmi-exporter\"}" timestamp=1689546660612
Jul 17 03:32:09 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:09.845Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"zookeeper_latency_avg_ms\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-Zookeeper-msh-dellr740-dt002\", endpoint_id=\"113\", module=\"DC-Zookeeper\", module_id=\"7\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", server=\"192.168.20.194:2181\"}" timestamp=1689546668156
Jul 17 03:32:09 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:09.920Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_compactionsFailedCount\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"assessment_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"38e639c1ffb12e33444c5eeed46e9713\", table=\"index_filename_assessment_hos_bucket\", type=\"index\"}" timestamp=1689546668804
Jul 17 03:32:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:10.420Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_lastMajorCompactionAge\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"1b0eadc46d8c5caaa1993208cc92777b\", table=\"troubleshooting_hos_bucket\", type=\"data\"}" timestamp=1689546668804
Jul 17 03:32:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:11.323Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_readRequestCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"assessment_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"91055c7f7731c83b1f2dde975c2f174d\", table=\"index_time_assessment_hos_bucket\", type=\"index\"}" timestamp=1689546670148
Jul 17 03:32:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:11.436Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_2_metric_getTime_max\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546670148
Jul 17 03:32:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:11.458Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_avgStoreFileAge\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"firewall_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"fb07700cfbe431ae4717fec4520565ee\", table=\"index_filename_firewall_hos_bucket\", type=\"index\"}" timestamp=1689546670148
Jul 17 03:32:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:11.480Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_194_1_metric_majorCompactionInputFileCount_mean\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546670148
Jul 17 03:32:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:11.619Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_2_metric_majorCompactionOutputSize_98th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546670148
Jul 17 03:32:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:11.639Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_default_metric_majorCompactionInputFileCount_99_9th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546670148
Jul 17 03:32:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:11.678Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_CHILD_LINK_metric_getTime_98th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546670148
Jul 17 03:32:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:11.700Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_default_metric_splitTime_90th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546670148
Jul 17 03:32:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:11.702Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_193_1_metric_compactionInputFileCount_mean\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546670148
Jul 17 03:32:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:11.728Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_session_record_hos_bucket_metric_majorCompactionInputSize_99_9th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546670148
Jul 17 03:32:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:11.761Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_avgStoreFileAge\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"ee67fa1edbe81be8d997d0da209e30d3\", table=\"knowledge_base_hos_bucket\", type=\"data\"}" timestamp=1689546670148
Jul 17 03:32:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:12.737Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_193_1_metric_flushTime_min\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546670148
Jul 17 03:32:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:12.817Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_session_record_hos_bucket_metric_deleteTime_98th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546670148
Jul 17 03:32:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:12.860Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_flushesQueuedCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"rtp_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"12da2e465ea17433fca2e4a9ecdfa346\", table=\"index_partfile_rtp_hos_bucket\", type=\"index\"}" timestamp=1689546670148
Jul 17 03:32:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:12.870Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_default_metric_compactionOutputFileCount_max\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546670148
Jul 17 03:32:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:12.888Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_numReferenceFiles\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"knowledge_base_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"1ee9566a08bea50a95dd060f332950cb\", table=\"index_partfile_knowledge_base_hos_bucket\", type=\"index\"}" timestamp=1689546670148
Jul 17 03:32:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:12.947Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_TASK_metric_incrementTime_25th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546670148
Jul 17 03:32:13 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:13.038Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_193_1_metric_majorCompactionTime_min\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546670148
Jul 17 03:32:13 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:13.062Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_minStoreFileAge\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"default\", project=\"OLAP\", project_id=\"3\", region=\"e2d375346aa16afac694d1080b6a27ff\", table=\"index_time_default\", type=\"index\"}" timestamp=1689546670148
Jul 17 03:32:13 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:13.077Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_heap_idle_bytes\", asset=\"twa-dellr740-dt002\", asset_id=\"39\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"OLAP-Node-Exporter-twa-dellr740-dt002\", endpoint_id=\"259\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546672064
Jul 17 03:32:13 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:13.087Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_194_1_metric_tableSize\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546670148
Jul 17 03:32:13 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:13.103Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_3_metric_compactionInputSize_mean\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546670148
Jul 17 03:32:13 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:13.105Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_STATS_metric_putBatchTime_median\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546670148
Jul 17 03:32:13 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:13.110Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_RenewalFailuresTotal\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"UgiMetrics\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546670148
Jul 17 03:32:13 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:13.145Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_193_1_metric_majorCompactionInputFileCount_90th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546670148
Jul 17 03:32:13 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:13.151Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_memStoreSize\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"ee67fa1edbe81be8d997d0da209e30d3\", table=\"knowledge_base_hos_bucket\", type=\"data\"}" timestamp=1689546670148
Jul 17 03:32:13 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:13.162Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_4_metric_deleteBatchTime_max\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546670148
Jul 17 03:32:13 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:13.183Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_1_metric_majorCompactionTime_min\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546670148
Jul 17 03:32:13 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:13.208Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_194_3_metric_filteredReadRequestCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546670148
Jul 17 03:32:13 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:13.228Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_tsg_table_report_result_metric_getTime_min\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546670148
Jul 17 03:32:13 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:13.273Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_194_1_metric_appendTime_99th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546670148
Jul 17 03:32:13 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:13.335Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_appendCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"1d4ca773a242a29a29b37966ec994837\", table=\"firewall_hos_bucket\", type=\"data\"}" timestamp=1689546670148
Jul 17 03:32:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:15.532Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"vgauthd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"simple\", type=\"service\"}" timestamp=1689546667046
Jul 17 03:32:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:15.695Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_labels\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"coredns-65dfb4fcf4-ss5kl\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"d909c7f6-8616-4c21-913b-225bb207a2ea\"}" timestamp=1689546668076
Jul 17 03:32:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:15.718Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"50\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546661230
Jul 17 03:32:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:16.081Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"123\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546661230
Jul 17 03:32:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:16.938Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"32\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546661230
Jul 17 03:32:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:19.466Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_device_id\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens8f1np1\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546677612
Jul 17 03:32:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:19.481Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_bytes_total\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens8f0np0\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546677612
Jul 17 03:32:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:19.498Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"122\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546677612
Jul 17 03:32:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:20.322Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_tasks_current\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"plymouth-quit.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689546670657
Jul 17 03:32:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:20.337Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_packets_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens6f0np0\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546668263
Jul 17 03:32:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:20.429Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546668263
Jul 17 03:32:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:20.536Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", container=\"dign-server-web\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Write\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546674690
Jul 17 03:32:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:20.617Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_dormant\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"lo\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546668263
Jul 17 03:32:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:20.706Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_memory_Mlocked_bytes\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546668263
Jul 17 03:32:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:20.736Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_compressed_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens8f0np0\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546668263
Jul 17 03:32:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:20.739Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"70\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546668263
Jul 17 03:32:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:20.748Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_errors_received_total\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_4\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546677612
Jul 17 03:32:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:20.754Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_avail_bytes\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda5\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", fstype=\"ext4\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/home\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546668263
Jul 17 03:32:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:20.785Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_address_assign_type\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth44c0947a\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546668263
Jul 17 03:32:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:20.895Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"nvmefc-boot-connections.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689546670657
Jul 17 03:32:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:20.914Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"10\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546677612
Jul 17 03:32:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:20.942Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-infiniband-umad8.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689546670657
Jul 17 03:32:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:20.965Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-podd69b2139_72e0_47ab_b29f_fa787b9e63b9.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"slice\"}" timestamp=1689546670657
Jul 17 03:32:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:21.162Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_created\", asset=\"twa-tsgx06\", asset_id=\"101\", configmap=\"prometheus-server\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546673949
Jul 17 03:32:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:21.199Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_node_status_condition\", asset=\"twa-tsgx06\", asset_id=\"101\", condition=\"MemoryPressure\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", node=\"localhost\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"unknown\"}" timestamp=1689546673949
Jul 17 03:32:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:21.205Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_owner\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", owner_is_controller=\"true\", owner_kind=\"Deployment\", owner_name=\"tsg-traffic-engine-numa-1-firewall\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-1-firewall-865db57bdd\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546673949
Jul 17 03:32:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:21.259Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"7\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546677612
Jul 17 03:32:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:21.281Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"24\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546677612
Jul 17 03:32:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:21.561Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_ulimits_soft\", asset=\"twa-tsgx06\", asset_id=\"101\", container=\"exporter\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", ulimit=\"max_open_files\"}" timestamp=1689546670010
Jul 17 03:32:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:21.666Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_ips\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", ip=\"10.42.0.63\", ip_family=\"4\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-kube-state-metrics-6b768c7bf8-fgd2z\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"3fd66c49-d621-40d3-b7b9-f758c2cf0c9c\"}" timestamp=1689546673949
Jul 17 03:32:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:21.715Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_restart_policy\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-alertmanager-6df7f457b6-z42km\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", type=\"Always\", uid=\"26d0147a-fc67-4990-96ac-7d6c8e980d26\"}" timestamp=1689546673828
Jul 17 03:32:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:21.749Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546668263
Jul 17 03:32:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:21.765Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_hwmon_chip_names\", asset=\"pcap-tsgx10\", asset_id=\"89\", chip=\"0000:80:01_2_0000:82:00_2\", chip_name=\"bnxt_en\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546668263
Jul 17 03:32:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:21.786Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"plymouth-switch-root.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"oneshot\", type=\"service\"}" timestamp=1689546670657
Jul 17 03:32:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:21.855Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2did-scsi\\\\x2dSDELL_PERC_H345_Adp_0041f2ff54318d2c2c0098d5adf0a7ce\\\\x2dpart3.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689546670657
Jul 17 03:32:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:21.953Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_labels\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.tsg-traffic-engine-numa-0.v2\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546673828
Jul 17 03:32:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:21.963Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_spec_replicas\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-1-firewall-5d7bfbbbfb\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546673828
Jul 17 03:32:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:21.982Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"lldpad.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"simple\", type=\"service\"}" timestamp=1689546670657
Jul 17 03:32:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:22.047Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"92\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546668263
Jul 17 03:32:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:22.099Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"42\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546668263
Jul 17 03:32:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:22.170Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"102\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546668263
Jul 17 03:32:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:22.189Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_readonly\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"shm\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/aebe77d3b5b78e4f598e7b51fd01ec5ef6e9c50bb0cb2dc1010f3ecfe3e703a8/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546668263
Jul 17 03:32:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:22.230Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"54\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546668263
Jul 17 03:32:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:22.251Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_sockets\", asset=\"pcap-tsgx10\", asset_id=\"89\", container=\"lb-tcp-443\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546667229
Jul 17 03:32:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:22.284Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"tmpfs\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/data/var/lib/kubelet/pods/73d5d2be-7515-4e91-9015-e0d525a9154e/volumes/kubernetes.io~projected/kube-api-access-76dln\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546668263
Jul 17 03:32:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:22.308Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"69\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546668263
Jul 17 03:32:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:22.381Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_multicast_packets_transmitted_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"mlx5_7\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546668263
Jul 17 03:32:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:23.293Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_address_assign_type\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethde0f386\", endpoint=\"OLAP-Node-Exporter-twa-dellr740-dt001\", endpoint_id=\"257\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546681406
Jul 17 03:32:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:23.489Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_carrier_total\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth7de59a65\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546674790
Jul 17 03:32:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:23.524Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-405e3b50\\\\x2d5fa3\\\\x2d4f39\\\\x2d9405\\\\x2dac337f946135-volume\\\\x2dsubpaths-config\\\\x2dvolume-telegraf-1.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689546678416
Jul 17 03:32:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:23.526Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"msh-tsgx09\", asset_id=\"55\", cpu=\"76\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546674790
Jul 17 03:32:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:23.569Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_up_changes_total\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens3f0np0\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546674790
Jul 17 03:32:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:23.581Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_disk_discards_merged_total\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"sda\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546674790
Jul 17 03:32:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:23.642Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_errs_total\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens8f1np1\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546674790
Jul 17 03:32:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:23.676Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db105\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546677019
Jul 17 03:32:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:23.711Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failcnt\", asset=\"msh-tsgx09\", asset_id=\"55\", container=\"lb-tcp-443\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546673056
Jul 17 03:32:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:23.809Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"msh-tsgx09\", asset_id=\"55\", container=\"prometheus-node-exporter\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"running\"}" timestamp=1689546671064
Jul 17 03:32:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:23.823Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"msh-tsgx09\", asset_id=\"55\", condition=\"false\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-delete-tsg-traffic-engine-numa-0-5rwhd\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"b14d723c-8283-4710-b1f5-86e0ceeae6a6\"}" timestamp=1689546680068
Jul 17 03:32:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:23.879Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_info\", address=\"bc:97:e1:0c:06:d1\", asset=\"msh-tsgx09\", asset_id=\"55\", broadcast=\"ff:ff:ff:ff:ff:ff\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens4f1np1\", duplex=\"unknown\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operstate=\"down\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546674790
Jul 17 03:32:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:23.916Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_limit_bytes\", asset=\"msh-tsgx09\", asset_id=\"55\", container=\"telegraf-security\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689546678586
Jul 17 03:32:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:23.929Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx09\", asset_id=\"55\", cpu=\"59\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546674790
Jul 17 03:32:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:23.932Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"msh-tsgx09\", asset_id=\"55\", condition=\"Progressing\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"prometheus-server\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"true\"}" timestamp=1689546680068
Jul 17 03:32:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:23.944Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"tmpfs\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/data/var/lib/kubelet/pods/350f7c1d-5c22-4da4-ab2d-ed8417c9cf5a/volumes/kubernetes.io~secret/kubernetes-dashboard-certs\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546674790
Jul 17 03:32:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:23.966Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_edac_correctable_errors_total\", asset=\"msh-tsgx09\", asset_id=\"55\", controller=\"1\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546674790
Jul 17 03:32:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:24.003Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_commands_failed_calls_total\", asset=\"msh-tsgx09\", asset_id=\"55\", cmd=\"replicaof\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546677019
Jul 17 03:32:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:24.016Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"tmpfs\", name=\"var-lib-kubelet-pods-350f7c1d\\\\x2d5c22\\\\x2d4da4\\\\x2dab2d\\\\x2ded8417c9cf5a-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2d672jh.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689546678416
Jul 17 03:32:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:24.138Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"116\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546680160
Jul 17 03:32:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:24.171Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_symbol_error_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"mlx5_1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546680160
Jul 17 03:32:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:24.187Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_packets_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens4f1np1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546680160
Jul 17 03:32:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:24.213Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"9\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546680160
Jul 17 03:32:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:24.605Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"ext4\", name=\"data-var-lib-kubelet-pods-692f1a13\\\\x2d48e0\\\\x2d47b7\\\\x2dad46\\\\x2da5b348bd402b-volume\\\\x2dsubpaths-config\\\\x2dvolume-telegraf-1.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689546671102
Jul 17 03:32:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:24.743Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_threads\", asset=\"msh-tsgx09\", asset_id=\"55\", container=\"configmap-reloader\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546666132
Jul 17 03:32:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:24.779Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_write_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/run\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546678566
Jul 17 03:32:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:24.809Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_stack_inuse_bytes\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546680160
Jul 17 03:32:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:24.831Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_daemonset_status_number_ready\", asset=\"pcap-tsgx12\", asset_id=\"117\", daemonset=\"kube-multus-ds\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546671122
Jul 17 03:32:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:24.856Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"coredns\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Sync\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546673261
Jul 17 03:32:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:24.882Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db120\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546681795
Jul 17 03:32:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:24.940Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_metadata_resource_version\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"chart-values-cm-policy-local-cache\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546680068
Jul 17 03:32:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:25.014Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-podb7741d71_12eb_4a84_9915_4d0db07d0a79.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"slice\"}" timestamp=1689546671102
Jul 17 03:32:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:25.019Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"cnibr0\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546674790
Jul 17 03:32:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:25.044Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx09\", asset_id=\"55\", container=\"dign-client\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Write\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546665719
Jul 17 03:32:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:25.142Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_swap\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"metrics-server\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546679208
Jul 17 03:32:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:25.164Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_annotations\", asset=\"pcap-tsgx12\", asset_id=\"117\", configmap=\"chart-content-mrzcpd-exporter\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546671122
Jul 17 03:32:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:25.295Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"pcap-tsgx12\", asset_id=\"117\", condition=\"true\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"cm-policy-local-cache-redis-master-69967b5879-8m6nj\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"ebdf9219-42ed-4a26-8500-14f2dacdb9fe\"}" timestamp=1689546671122
Jul 17 03:32:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:25.334Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"109\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546680160
Jul 17 03:32:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:25.338Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"82\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546680160
Jul 17 03:32:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:25.357Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_threads_max\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689546682879
Jul 17 03:32:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:25.387Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"21\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546680160
Jul 17 03:32:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:25.418Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"pcap-tsgx12\", asset_id=\"117\", condition=\"Progressing\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"metrics-server\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"true\"}" timestamp=1689546671122
Jul 17 03:32:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:25.423Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_errors_received_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"mlx5_9\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546680160
Jul 17 03:32:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:25.435Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"lo\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546680160
Jul 17 03:32:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:25.442Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_os_version\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", id=\"rocky\", id_like=\"rhel centos fedora\", module=\"TSG-OS\", module_id=\"77\", name=\"TSG-OS\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546680160
Jul 17 03:32:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:25.454Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"100\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546680160
Jul 17 03:32:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:25.469Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_init_container_status_terminated_reason\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"firewall-init\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", pod=\"tsg-traffic-engine-numa-1-firewall-cdbcd8f4f-k98cr\", project=\"TSG-OS\", project_id=\"7\", reason=\"Completed\", service=\"prometheus-kube-state-metrics\", uid=\"a27a1ebc-016c-4998-a3f3-85e865dfcd16\"}" timestamp=1689546671122
Jul 17 03:32:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:25.491Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_info\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"helm\", container_id=\"containerd://878cada3ebbde44c514fb294946c66da47425fddcb91d8907c3a13db10a87517\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", image=\"docker.io/rancher/klipper-helm:v0.8.0-build20230510\", image_id=\"sha256:6f42df210d7faa0b6eb56f5f28bb061930c7834af1e7dd4b3586108ec6b1485b\", image_spec=\"rancher/klipper-helm:v0.8.0-build20230510\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-tsg-traffic-engine-numa-0-mdqrn\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"86c61b36-b5a0-422f-be56-b68f56297496\"}" timestamp=1689546671122
Jul 17 03:32:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:25.510Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"105\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546680160
Jul 17 03:32:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:25.516Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"configmap-reloader\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Total\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546681066
Jul 17 03:32:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:25.518Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_mem_not_counted_for_eviction_bytes\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546681795
Jul 17 03:32:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:25.531Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"17\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546680160
Jul 17 03:32:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:25.535Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"26\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546680160
Jul 17 03:32:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:25.538Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-b40f3139a985682e40ec310679e88017429c5d22e99d3b0f8b73c9d6ea1009c6-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689546671102
Jul 17 03:32:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:25.540Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"49\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546680160
Jul 17 03:32:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:25.550Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_up\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"vethe56cbeac\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546680160
Jul 17 03:32:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:25.567Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_avail_bytes\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"shm\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/2e438e2d86de3c658a3d680f577f8e1e8e7876c5c1cb1fb878b12b31c449954a/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546680160
Jul 17 03:32:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:25.576Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_sockstat_FRAG_memory\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546680160
Jul 17 03:32:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:25.577Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"57\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546680160
Jul 17 03:32:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:25.584Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_name_assign_type\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"eno33np0\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546680160
Jul 17 03:32:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:25.597Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"70\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546680160
Jul 17 03:32:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:25.600Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db36\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546681795
Jul 17 03:32:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:25.612Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_tasks_max\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-quotacheck.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689546671102
Jul 17 03:32:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:25.615Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-pode8a13065_a344_44f4_a0e1_7369c47522e2.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"slice\"}" timestamp=1689546671102
Jul 17 03:32:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:25.628Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth64615143\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546680160
Jul 17 03:32:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:25.663Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_bytes_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens1f0np0\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546680160
Jul 17 03:32:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:25.679Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-99e215df\\\\x2d15ea\\\\x2d44bd\\\\x2d824d\\\\x2d42fb75c6e74b-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2ds24s9.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689546671102
Jul 17 03:32:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:25.773Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"109\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546680160
Jul 17 03:32:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:25.789Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"25\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546680160
Jul 17 03:32:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:25.848Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2duuid-ad80db0e\\\\x2d023b\\\\x2d4d11\\\\x2d989c\\\\x2d06b86e9bfff0.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689546671102
Jul 17 03:32:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:25.865Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-bb4505d74cdfdab9d73c1fb267be2c89d63fd646da06fa83cc97944fef8187d9-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689546671102
Jul 17 03:32:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:26.418Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"58\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546675810
Jul 17 03:32:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:26.418Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Pkt_Outbound\", app_name=\"sapp4\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", vsysId=\"1\"}" timestamp=1689546675421
Jul 17 03:32:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:26.441Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db152\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546683549
Jul 17 03:32:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:26.483Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_spec_strategy_rollingupdate_max_surge\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"dashboard-metrics-scraper\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546678744
Jul 17 03:32:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:26.748Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_cpu_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-pod80e5751c_f8e9_49bd_ae9e_664fe9b134da.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"slice\"}" timestamp=1689546673907
Jul 17 03:32:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:26.765Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2duuid-79d91b39\\\\x2df221\\\\x2d4fb4\\\\x2d9b48\\\\x2d8293e9158b23.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689546673907
Jul 17 03:32:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:27.662Z caller=dedupe.go:112 component=remote level=error remote_name=52db6f url=http://10.10.10.159:10090/nz-agent/prometheus/proxy/api/v1/write msg="non-recoverable error" count=1000 exemplarCount=0 err="server returned HTTP status 400 : out of order sample"
Jul 17 03:32:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:27.737Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_last_seen\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"dign-client\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546684861
Jul 17 03:32:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:27.772Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_metadata_resource_version\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"ipmi-exporter\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546675682
Jul 17 03:32:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:27.827Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"87\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546682882
Jul 17 03:32:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:28.529Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_created\", asset=\"msh-tsgx10\", asset_id=\"69\", configmap=\"cm-policy-local-cache-redis-configuration\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546675682
Jul 17 03:32:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:28.607Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_last_seen\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689546684914
Jul 17 03:32:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:28.679Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_sector_reads_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/sys/fs/cgroup\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546683185
Jul 17 03:32:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:28.687Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_sockets\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"prometheus-alertmanager-configmap-reload\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546674874
Jul 17 03:32:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:28.702Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"41\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546682882
Jul 17 03:32:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:28.709Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link_mode\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"vethc9ac6a15\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546682882
Jul 17 03:32:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:31.568Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_hwmon_temp_celsius\", asset=\"msh-tsgx05\", asset_id=\"67\", chip=\"pci0000:00_0000:00:18_3\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", sensor=\"temp3\", service=\"prometheus-node-exporter\"}" timestamp=1689546680939
Jul 17 03:32:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:31.606Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_carrier_total\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"virtio_dign_c\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546680939
Jul 17 03:32:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:31.616Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_net_dev_group\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"vethaec54176\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546680939
Jul 17 03:32:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:31.646Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_packets_total\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"cadvisor\", interface=\"ens4f2np2\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546681869
Jul 17 03:32:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:31.667Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"90\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546680939
Jul 17 03:32:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:31.733Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_resident_memory_bytes\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"udisks2.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689546681015
Jul 17 03:32:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:31.772Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_labels\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.prometheus.v1\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546690436
Jul 17 03:32:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:31.791Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"eno33np0\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546679093
Jul 17 03:32:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:31.806Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"home.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689546686907
Jul 17 03:32:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:31.812Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"37\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546680791
Jul 17 03:32:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:31.825Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:80-0000:80:01.1-0000:81:00.0-host0-target0:2:0-0:2:0:0-block-sda-sda3.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689546682644
Jul 17 03:32:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:31.855Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-d4c7fbf81264a70084798a468d5a4020c351d057fad0b11ee2e45616ab9f726a-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689546682644
Jul 17 03:32:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:31.882Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"iscsid.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"service\"}" timestamp=1689546681015
Jul 17 03:32:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:31.893Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_gc_duration_seconds\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", quantile=\"0\", service=\"ipmi-exporter\"}" timestamp=1689546679925
Jul 17 03:32:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:31.908Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"91\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546680791
Jul 17 03:32:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:31.914Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"7\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546679093
Jul 17 03:32:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:31.926Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_created\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", job_name=\"helm-install-ipmi-exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546690436
Jul 17 03:32:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:31.959Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_node_status_allocatable\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", node=\"localhost\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", resource=\"cpu\", service=\"prometheus-kube-state-metrics\", unit=\"core\"}" timestamp=1689546690436
Jul 17 03:32:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:31.964Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"configmap-reloader\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Write\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546668161
Jul 17 03:32:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:32.000Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-ipmi-exporter-h7mdw\", project=\"TSG-OS\", project_id=\"7\", reason=\"UnexpectedAdmissionError\", service=\"prometheus-kube-state-metrics\", uid=\"14488225-fd95-4f38-a9af-d4af570ca0f1\"}" timestamp=1689546690436
Jul 17 03:32:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:32.007Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"twa-tsgx03\", asset_id=\"95\", condition=\"unknown\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-kube-state-metrics-6b768c7bf8-8z25n\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"0f76af51-4cb4-40c5-8ff5-c445f8b7b608\"}" timestamp=1689546690436
Jul 17 03:32:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:32.020Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_owner\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", owner_is_controller=\"true\", owner_kind=\"Deployment\", owner_name=\"prometheus-kube-state-metrics\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"prometheus-kube-state-metrics-6b768c7bf8\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546690436
Jul 17 03:32:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:32.022Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_queue_length\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens4f0np0\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546679093
Jul 17 03:32:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:32.047Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"86\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546679093
Jul 17 03:32:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:32.061Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Failed\", pod=\"helm-install-configmap-reloader-p2p4j\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"f81c8fb3-e0bf-42af-a2a5-7e8f904d8466\"}" timestamp=1689546690436
Jul 17 03:32:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:32.066Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"50\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546680939
Jul 17 03:32:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:32.066Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-ens2f1np1.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689546686907
Jul 17 03:32:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:32.080Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_constraint_errors_received_total\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_1\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546679093
Jul 17 03:32:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:32.089Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_usage_bytes\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"exporter\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546670751
Jul 17 03:32:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:32.098Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"msh-tsgx11\", asset_id=\"111\", condition=\"Progressing\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"cm-policy-local-cache-redis-master\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"false\"}" timestamp=1689546679276
Jul 17 03:32:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:32.105Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_drop_total\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens4f2np2\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546680791
Jul 17 03:32:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:32.145Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"twa-tsgx03\", asset_id=\"95\", condition=\"true\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-node-exporter-dd5xk\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"18031228-abd9-478c-bf5c-0acdbff9f2e6\"}" timestamp=1689546690436
Jul 17 03:32:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:32.147Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"configmap-reloader\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Total\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546668161
Jul 17 03:32:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:32.150Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_protocol_type\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth3448a07b\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546680791
Jul 17 03:32:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:32.153Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"87\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546680939
Jul 17 03:32:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:32.156Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-ttyS3.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689546682644
Jul 17 03:32:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:32.173Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"overlay\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-95d5b60888ba291b4d576ad8bf0b3f4ea40c0dc6b50f0b3efcf92f20d93cc451-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689546681015
Jul 17 03:32:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:32.200Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_spec_completions\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", job_name=\"helm-install-tsg-traffic-engine-numa-0\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546679276
Jul 17 03:32:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:32.215Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"46\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546679093
Jul 17 03:32:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:32.226Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"dign-client\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Async\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546681016
Jul 17 03:32:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:32.237Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-virtual-net-veth244c5093.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689546686907
Jul 17 03:32:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:32.240Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"msh-tsgx05\", asset_id=\"67\", condition=\"Progressing\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"metrics-server\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"true\"}" timestamp=1689546684068
Jul 17 03:32:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:32.245Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"msh-tsgx11\", asset_id=\"111\", condition=\"unknown\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-cm-policy-local-cache-k4zkb\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"9773a9ea-4c23-4577-b88b-c11f7277b019\"}" timestamp=1689546679276
Jul 17 03:32:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:32.266Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_dormant\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"eno2\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546679093
Jul 17 03:32:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:32.272Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_free_bytes\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"shm\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/61933424a8b84f33d9f7ac8f9f985f2a9248972a97f5ab81d8fb68bf4927c61d/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546679093
Jul 17 03:32:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:32.278Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_errs_total\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth15723f66\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546680791
Jul 17 03:32:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:32.284Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"4\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546680791
Jul 17 03:32:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:32.284Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"21\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546679093
Jul 17 03:32:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:32.290Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_frame_total\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"eno1\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546679093
Jul 17 03:32:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:32.300Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db48\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546676711
Jul 17 03:32:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:32.304Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-76e087a56cab779cf96a899874139822324870dc5f2cec4e3ac4557ffdaadea5.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"scope\"}" timestamp=1689546686907
Jul 17 03:32:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:32.374Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"118\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546680939
Jul 17 03:32:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:32.391Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"42\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546680939
Jul 17 03:32:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:32.464Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"multi-user.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"target\"}" timestamp=1689546681015
Jul 17 03:32:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:32.482Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"tsg_master_T_success_log\", app_name=\"tsg_master\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"traffic-engine\", line_name=\"TRANSACTION-RECORD\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", table_name=\"sendlog\", vsysId=\"1\"}" timestamp=1689546689611
Jul 17 03:32:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:32.496Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_id\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth39378f16\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546679093
Jul 17 03:32:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:32.510Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_fifo_total\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth662ec9f2\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546679093
Jul 17 03:32:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:32.524Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_multicast_total\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"br_dign_c\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546679093
Jul 17 03:32:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:32.555Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_created\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"chart-values-prometheus\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546690436
Jul 17 03:32:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:32.566Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"initrd.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"target\"}" timestamp=1689546682644
Jul 17 03:32:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:32.570Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"116\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546679093
Jul 17 03:32:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:32.573Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_labels\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"kubernetes-dashboard-66dd5b456\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546679276
Jul 17 03:32:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:32.613Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"121\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546679093
Jul 17 03:32:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:32.617Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-1ae789f3\\\\x2d8760\\\\x2d4836\\\\x2d9727\\\\x2d170790799776-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2dzpppq.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689546682644
Jul 17 03:32:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:32.625Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Unknown_Pkt\", app_name=\"sapp4\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", vsysId=\"1\"}" timestamp=1689546680718
Jul 17 03:32:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:32.631Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-sda3.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689546682644
Jul 17 03:32:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:32.643Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_packets_total\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", interface=\"eth0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689546681638
Jul 17 03:32:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:32.663Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_spec_replicas\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"tsg-traffic-engine-numa-1-firewall\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546679276
Jul 17 03:32:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:32.715Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_changes_total\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"br_dign_c\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546680791
Jul 17 03:32:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:32.741Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_namespace_annotations\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546690436
Jul 17 03:32:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:32.746Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_load_average_10s\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"firewall\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689546673530
Jul 17 03:32:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:32.803Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_client_recent_max_output_buffer_bytes\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546676711
Jul 17 03:32:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:32.831Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"118\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546680791
Jul 17 03:32:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:32.891Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", state=\"stopped\"}" timestamp=1689546671938
Jul 17 03:32:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:32.892Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_restart_policy\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"mrzcpd-exporter-5cf7f967cc-hrwcm\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", type=\"Always\", uid=\"de0f6c6e-3a5e-493f-bc37-51ac5bb8c9a3\"}" timestamp=1689546679276
Jul 17 03:32:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:32.918Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_device_id\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"vethf7a1468f\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546680791
Jul 17 03:32:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:32.961Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-pushgateway-bd476bcf9-5mpjx\", project=\"TSG-OS\", project_id=\"7\", reason=\"NodeAffinity\", service=\"prometheus-kube-state-metrics\", uid=\"f2075fac-2ce7-4f3c-b94c-4c05e2de4805\"}" timestamp=1689546679894
Jul 17 03:32:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:33.044Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"56\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546680791
Jul 17 03:32:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:33.082Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_fifo_total\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens4f2np2\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546679093
Jul 17 03:32:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:33.097Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_drop_total\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethc8d1fba9\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546679093
Jul 17 03:32:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:33.107Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"scrape_duration_seconds\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_instance=\"localhost\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546682009
Jul 17 03:32:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:33.120Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"3\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546680605
Jul 17 03:32:35 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:35.848Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"pcap-tsgx09\", asset_id=\"83\", cpu=\"10\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx09\", endpoint_id=\"477\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546684944
Jul 17 03:32:38 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:38.502Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_gc_duration_seconds\", agent_name=\"msh-kvm-nz001\", application=\"blackbox-exporter\", datacenter=\"MSH\", instance=\"127.0.0.1:19115\", quantile=\"0\"}" timestamp=1689546697994
Jul 17 03:32:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:41.395Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_max_fds\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"getty@tty1.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689546695764
Jul 17 03:32:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:41.904Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_system_seconds_total\", asset=\"twa-tsgx04\", asset_id=\"97\", container=\"ipmi-exporter\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546685903
Jul 17 03:32:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:41.961Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"55\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546691647
Jul 17 03:32:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:41.964Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_net_dev_group\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"eno33np0\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546691647
Jul 17 03:32:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:41.989Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_compressed_total\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth0ed14f65\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546691647
Jul 17 03:32:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:42.033Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db206\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546693827
Jul 17 03:32:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:42.083Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_10_193_1_metric_deleteTime_75th_percentile\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546699756
Jul 17 03:32:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:42.457Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", cpu=\"83\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546696242
Jul 17 03:32:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:42.477Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-tmpfiles-setup.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689546700671
Jul 17 03:32:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:42.485Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_spec_volumes_persistentvolumeclaims_readonly\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", persistentvolumeclaim=\"prometheus-alertmanager\", pod=\"prometheus-alertmanager-6df7f457b6-m8cl7\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"98daa3ea-45f0-41c2-be94-512fb3e2a016\", volume=\"storage-volume\"}" timestamp=1689546695154
Jul 17 03:32:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:42.585Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_labels\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.mrzcpd-exporter.v1\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546695154
Jul 17 03:32:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:42.599Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failcnt\", asset=\"pcap-tsgx01\", asset_id=\"71\", container=\"exporter\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546693840
Jul 17 03:32:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:42.607Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", cpu=\"3\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546696242
Jul 17 03:32:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:42.620Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-095d5887\\\\x2d0ff1\\\\x2d468e\\\\x2d9463\\\\x2da3765c458177-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2dx4z8p.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689546700671
Jul 17 03:32:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:42.649Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_id\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens7f0np0\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546696242
Jul 17 03:32:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:42.663Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Tcp_Bit\", app_name=\"sapp4\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", vsysId=\"1\"}" timestamp=1689546689178
Jul 17 03:32:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:42.696Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_virtual_memory_max_bytes\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"rsyslog.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689546700671
Jul 17 03:32:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:42.736Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"tmpfs\", name=\"data-var-lib-kubelet-pods-d20d4034\\\\x2d164e\\\\x2d4450\\\\x2da851\\\\x2d056ee84ef97e-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2dfjph5.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689546700671
Jul 17 03:32:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:42.752Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_spec_replicas\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"tsg-traffic-engine-numa-0-firewall\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546695154
Jul 17 03:32:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:42.769Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_annotations\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-1-firewall-77bd89f644\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546695154
Jul 17 03:32:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:42.781Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens3f0np0\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546696242
Jul 17 03:32:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:42.827Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_cache\", asset=\"pcap-tsgx01\", asset_id=\"71\", container=\"lb-tcp-443\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546683422
Jul 17 03:32:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:42.842Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth017375a5\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546696242
Jul 17 03:32:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:42.859Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-sda4.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689546700671
Jul 17 03:32:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:42.873Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_spec_type\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"ClusterIP\", uid=\"608dc4ed-eac0-4c84-a522-931a51275eca\"}" timestamp=1689546695154
Jul 17 03:32:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:42.883Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-module-fuse.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689546700671
Jul 17 03:32:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:42.899Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", container=\"exporter\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"stopped\"}" timestamp=1689546693840
Jul 17 03:32:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:42.933Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_fifo_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth696e0e5f\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546696242
Jul 17 03:32:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:43.106Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_mtu_bytes\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth7183a5ce\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546696242
Jul 17 03:32:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:43.244Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_files\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"shm\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/b975f5bfa7085daf7ea4cf8c6128f56b14518d193379482563f1392151c1cd72/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546696242
Jul 17 03:32:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:43.249Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", cpu=\"108\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546696242
Jul 17 03:32:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:43.251Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-alertmanager-6df7f457b6-m8cl7\", project=\"TSG-OS\", project_id=\"7\", reason=\"NodeLost\", service=\"prometheus-kube-state-metrics\", uid=\"98daa3ea-45f0-41c2-be94-512fb3e2a016\"}" timestamp=1689546695154
Jul 17 03:32:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:43.259Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_readonly\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"shm\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/af190c8e01b5d9dc3d0e83d340e7d3c7cc03e8e703ecce75aa0a3882d8ac5716/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546696242
Jul 17 03:32:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:43.279Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_up_changes_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"lo\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546696242
Jul 17 03:32:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:43.309Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_repl_backlog_first_byte_offset\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546694838
Jul 17 03:32:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:43.420Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", cpu=\"48\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546696242
Jul 17 03:32:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:43.552Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_packets_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"cadvisor\", interface=\"ens2f1np1\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546698943
Jul 17 03:32:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:43.564Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_info\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"kubernetes-dashboard-key-holder\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546695154
Jul 17 03:32:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:43.617Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-sda.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689546700671
Jul 17 03:32:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:43.618Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", phase=\"Failed\", pod=\"prometheus-pushgateway-bd476bcf9-scrs2\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"1d9c1047-30bc-4de8-bd00-07e7917cd6b7\"}" timestamp=1689546695154
Jul 17 03:32:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:43.633Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"10\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546696242
Jul 17 03:32:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:43.635Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dracut-shutdown-onfailure.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"service\"}" timestamp=1689546700671
Jul 17 03:32:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:43.669Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", container=\"dign-server-dns\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Read\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546694918
Jul 17 03:32:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:43.684Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_limit_bytes\", asset=\"pcap-tsgx01\", asset_id=\"71\", container=\"mrapm-device\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546700951
Jul 17 03:32:44 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:44.083Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"mysql_global_status_commands_total\", asset=\"msh-dellr740-olap002\", asset_id=\"13\", command=\"assign_to_keycache\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Mariadb-msh-dellr740-olap002\", endpoint_id=\"69\", module=\"NC-Mariadb\", module_id=\"43\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546702786
Jul 17 03:32:48 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:48.687Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_hbase_table_namespace_metric_flushMemstoreSize_75th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546707438
Jul 17 03:32:48 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:48.729Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_maxFlushQueueSize\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"firewall_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"004874e913c7921a27c0297c5bb8d8c9\", table=\"index_filename_firewall_hos_bucket\", type=\"index\"}" timestamp=1689546707438
Jul 17 03:32:48 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:48.744Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_getCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"default\", project=\"OLAP\", project_id=\"3\", region=\"668ca3489d8c6fec92a10db8b81f6e41\", table=\"index_partfile_default\", type=\"index\"}" timestamp=1689546707438
Jul 17 03:32:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:49.098Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_minStoreFileAge\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"assessment_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"73b5ff36af1eb934d5c67441ee3e9bca\", table=\"index_time_assessment_hos_bucket\", type=\"index\"}" timestamp=1689546707438
Jul 17 03:32:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:49.132Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_readRequestCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"session_record_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"3c97dedf134d1e49c38f42e6958b7ec5\", table=\"index_time_session_record_hos_bucket\", type=\"index\"}" timestamp=1689546707438
Jul 17 03:32:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:49.190Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_1_metric_majorCompactionInputFileCount_25th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546707438
Jul 17 03:32:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:49.217Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_numBytesCompactedCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"session_record_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"c466e5d78e3f68e42ef3c5a869096c65\", table=\"index_partfile_session_record_hos_bucket\", type=\"index\"}" timestamp=1689546707438
Jul 17 03:32:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:49.285Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_writeRequestCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"rtp_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"5815b1ad7d1924729ecaa0e272da2c37\", table=\"index_time_rtp_hos_bucket\", type=\"index\"}" timestamp=1689546707438
Jul 17 03:32:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:49.355Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_193_1_metric_compactionInputSize_99th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546707438
Jul 17 03:32:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:49.424Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_lastMajorCompactionAge\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"proxy_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"12c8bac854969dda8707b0a6c52163c3\", table=\"index_time_proxy_hos_bucket\", type=\"index\"}" timestamp=1689546707438
Jul 17 03:32:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:49.446Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_194_3_metric_scanSize_95th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546707438
Jul 17 03:32:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:49.531Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"104\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546701797
Jul 17 03:32:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:49.541Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_4_metric_appendTime_75th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546707438
Jul 17 03:32:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:49.645Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_namespace_created\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546699668
Jul 17 03:32:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:49.662Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx10\", asset_id=\"109\", cpu=\"50\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546701797
Jul 17 03:32:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:49.675Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_id\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth83674fd8\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546701797
Jul 17 03:32:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:49.691Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_created\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"mrzcpd-exporter-5cf7f967cc\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546699668
Jul 17 03:32:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:49.704Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_compactionsQueuedCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"troubleshooting_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"8d05b49601aeb317102c899158fd3ce4\", table=\"index_partfile_troubleshooting_hos_bucket\", type=\"index\"}" timestamp=1689546707438
Jul 17 03:32:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:49.712Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"scrape_duration_seconds\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_instance=\"localhost\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546699050
Jul 17 03:32:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:49.722Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_194_2_metric_compactionOutputFileCount_25th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546707438
Jul 17 03:32:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:49.738Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_194_4_metric_scanSize_98th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546707438
Jul 17 03:32:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:49.856Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx10\", asset_id=\"109\", cpu=\"59\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546701797
Jul 17 03:32:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:49.870Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_compressed_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens4f1np1\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546706709
Jul 17 03:32:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:49.885Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-ba106c44\\\\x2dd5c3\\\\x2d446a\\\\x2db10b\\\\x2d530b2492f73f-volume\\\\x2dsubpaths-config\\\\x2dvolume-telegraf-2.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689546701057
Jul 17 03:32:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:49.896Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"80\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546706709
Jul 17 03:32:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:49.897Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_scaling_frequency_max_hertz\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546706709
Jul 17 03:32:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:49.901Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_maxStoreFileAge\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"troubleshooting_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"572ae852594a05485a654cdf44db2bd2\", table=\"index_partfile_troubleshooting_hos_bucket\", type=\"index\"}" timestamp=1689546707438
Jul 17 03:32:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:49.917Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2duuid-7a413e4b\\\\x2d8577\\\\x2d45c9\\\\x2d825d\\\\x2d4b35033503b7.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689546701057
Jul 17 03:32:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:49.919Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"15\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546706709
Jul 17 03:32:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:49.935Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_frequency_max_hertz\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", cpu=\"13\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-olap003\", endpoint_id=\"235\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"yarn_resourcemanager,zookeeper,nacos,kafka,spark,druid_data,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546709714
Jul 17 03:32:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:49.936Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_net_dev_group\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth75d96d64\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546706709
Jul 17 03:32:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:49.951Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"scrape_samples_post_metric_relabeling\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-olap003\", endpoint_id=\"235\", instance=\"192.168.20.223:9100\", job=\"235\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"yarn_resourcemanager,zookeeper,nacos,kafka,spark,druid_data,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546709714
Jul 17 03:32:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:49.972Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_writes_merged_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/shm\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546703239
Jul 17 03:32:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:50.108Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"cadvisor_version_info\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", kernelVersion=\"5.17.15-1.el8.x86_64\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", osVersion=\"TSG-OS v23.07-rc1-079e964 (TSGXNXR620G40R01P0906)\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546707561
Jul 17 03:32:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:50.130Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens2f0np0\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546706709
Jul 17 03:32:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:50.183Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", phase=\"Failed\", pod=\"prometheus-server-6ff784d758-jgfzf\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"dc3f3ccc-197b-483a-887e-f7d934ff1ed5\"}" timestamp=1689546699668
Jul 17 03:32:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:50.216Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db95\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546706625
Jul 17 03:32:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:50.218Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_errors_total\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", interface=\"ens4f0np0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546695973
Jul 17 03:32:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:50.391Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_packets_dropped_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", interface=\"ens6f1np1\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546705663
Jul 17 03:32:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:50.456Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"46\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546706709
Jul 17 03:32:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:50.499Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2dpartlabel-TSG\\\\x2dOS\\\\x2dUPDATE.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689546706554
Jul 17 03:32:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:50.509Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_start_time_seconds\", asset=\"twa-tsgx08\", asset_id=\"105\", container=\"configmap-reloader\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546707561
Jul 17 03:32:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:50.543Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_spec_volumes_persistentvolumeclaims_info\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", persistentvolumeclaim=\"prometheus-server\", pod=\"prometheus-server-6ff784d758-jgfzf\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"dc3f3ccc-197b-483a-887e-f7d934ff1ed5\", volume=\"storage-volume\"}" timestamp=1689546699668
Jul 17 03:32:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:50.659Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"twa-tsgx08\", asset_id=\"105\", container=\"exporter\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"uninterruptible\"}" timestamp=1689546704025
Jul 17 03:32:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:50.685Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_labels\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"kubernetes-dashboard-66dd5b456\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546701568
Jul 17 03:32:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:50.748Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link_mode\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth75d96d64\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546706709
Jul 17 03:32:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:50.770Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", phase=\"Running\", pod=\"systemd-exporter-7f549d9485-qjwvl\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"d1333293-1dd6-450f-8c65-92ef3d57012b\"}" timestamp=1689546699668
Jul 17 03:32:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:50.795Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_state_id\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546701797
Jul 17 03:32:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:50.814Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_colls_total\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethec197529\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546701797
Jul 17 03:32:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:50.833Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_virtual_memory_max_bytes\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-logind.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689546701057
Jul 17 03:32:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:50.871Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_up_changes_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens7f1np1\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546706709
Jul 17 03:32:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:50.895Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"108\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546706709
Jul 17 03:32:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:50.902Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_memory_MemAvailable_bytes\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-olap003\", endpoint_id=\"235\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"yarn_resourcemanager,zookeeper,nacos,kafka,spark,druid_data,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546709714
Jul 17 03:32:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:50.918Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sysstat.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"service\"}" timestamp=1689546706554
Jul 17 03:32:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:50.922Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_errs_total\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth34c9090\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-olap003\", endpoint_id=\"235\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"yarn_resourcemanager,zookeeper,nacos,kafka,spark,druid_data,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546709714
Jul 17 03:32:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:50.943Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_carrier_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth5b68c36f\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546706709
Jul 17 03:32:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:51.000Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_virtual_memory_bytes\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-logind.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689546701057
Jul 17 03:32:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:51.129Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"dign-server-dns\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Total\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546697910
Jul 17 03:32:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:51.266Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_init_container_status_running\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"firewall-init\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", pod=\"tsg-traffic-engine-numa-1-firewall-58f9975657-n9n7b\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"ba106c44-d5c3-446a-b10b-530b2492f73f\"}" timestamp=1689546699668
Jul 17 03:32:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:51.276Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_labels\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_endpoint=\"cm-policy-local-cache-redis-metrics\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546701568
Jul 17 03:32:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:51.306Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_spec_replicas\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-1-firewall-5c6765fb69\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546699668
Jul 17 03:32:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:51.321Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"113\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546706709
Jul 17 03:32:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:51.346Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_frequency_min_hertz\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"27\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546706709
Jul 17 03:32:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:51.350Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_scaling_frequency_hertz\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"24\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546706709
Jul 17 03:32:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:51.372Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"overlay\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-4b9263181e793663d28e43688a0d5d95b4abb8820b932ec2f24053e8384bb493-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689546706554
Jul 17 03:32:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:51.404Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_scaling_frequency_max_hertz\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"26\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546706709
Jul 17 03:32:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:51.430Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"twa-tsgx08\", asset_id=\"105\", container=\"telegraf-security\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", failure_type=\"pgmajfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"container\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689546701229
Jul 17 03:32:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:51.430Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2did-scsi\\\\x2dSDELL_PERC_H345_Adp_009195b3b23903272c00c14683f0a7ce\\\\x2dpart1.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689546706554
Jul 17 03:32:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:51.449Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_oom_events_total\", asset=\"twa-tsgx08\", asset_id=\"105\", container=\"lb-tcp-443\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546705212
Jul 17 03:32:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:51.460Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_spec_volumes_persistentvolumeclaims_info\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", persistentvolumeclaim=\"prometheus-alertmanager\", pod=\"prometheus-alertmanager-6df7f457b6-vgl6l\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"9fc73d80-3d8a-43be-8ada-97bd28ed31fd\", volume=\"storage-volume\"}" timestamp=1689546699668
Jul 17 03:32:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:51.634Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"49\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546706709
Jul 17 03:32:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:51.682Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_receive_switch_relay_errors_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_9\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546706709
Jul 17 03:32:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:51.699Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"44\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546706709
Jul 17 03:32:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:51.701Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", cpu=\"0\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-olap003\", endpoint_id=\"235\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"yarn_resourcemanager,zookeeper,nacos,kafka,spark,druid_data,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546709714
Jul 17 03:32:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:51.707Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_inodes_free\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/data/var/lib/kubelet/pods/3a7372b8-22f7-4934-bc38-fba12a319dfc/volumes/kubernetes.io~projected/kube-api-access-x5c6f\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546703239
Jul 17 03:32:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:51.725Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db11\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546707556
Jul 17 03:32:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:51.728Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_writes_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/0500205352582fe55f0c9607f3e3848e1f23c3fd83667094351e6d099fc71fcc/shm\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546703239
Jul 17 03:32:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:51.774Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_info\", asset=\"twa-tsgx08\", asset_id=\"105\", container=\"coredns\", container_id=\"containerd://6996958b3a370e9da98ab57109d5cc0b6c33f107967b6635c78106d3bffbfaf9\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", image=\"docker.io/rancher/mirrored-coredns-coredns:1.10.1\", image_id=\"sha256:ead0a4a53df89fd173874b46093b6e62d8c72967bbf606d672c9e8c9b601a4fc\", image_spec=\"rancher/mirrored-coredns-coredns:1.10.1\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"coredns-55886d8f9d-vf9r8\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"fd4b4291-55db-4894-9795-6abc7f3024db\"}" timestamp=1689546701568
Jul 17 03:32:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:51.789Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db76\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546707556
Jul 17 03:32:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:51.851Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_bytes_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", interface=\"ens2f1np1\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546705663
Jul 17 03:32:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:51.868Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_terminated\", asset=\"twa-tsgx08\", asset_id=\"105\", container=\"configmap-reloader\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"configmap-reloader-6c55bbc49f-pvgz5\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"b2468ee7-2c6b-4e72-8bc5-bd363342dc00\"}" timestamp=1689546701568
Jul 17 03:32:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:52.535Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_receive_switch_relay_errors_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"mlx5_6\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546709337
Jul 17 03:32:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:52.551Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_vmstat_pgfault\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546709337
Jul 17 03:32:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:52.570Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", container=\"firewall\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", state=\"uninterruptible\"}" timestamp=1689546704539
Jul 17 03:32:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:52.604Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_bytes_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"cadvisor\", interface=\"br_dign_s\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546709420
Jul 17 03:32:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:52.620Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"pcap-tsgx11\", asset_id=\"115\", condition=\"unknown\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"svclb-traefik-644fb0c0-7cq8s\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"ea6cc882-864f-480f-b3a6-4af3bfdaa023\"}" timestamp=1689546704478
Jul 17 03:32:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:52.675Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-pod93005333_2bbb_4ce3_a6b6_a5b632269832.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"slice\"}" timestamp=1689546705496
Jul 17 03:32:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:52.938Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_owner\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", owner_is_controller=\"true\", owner_kind=\"Job\", owner_name=\"helm-install-tsg-traffic-engine-numa-0\", pod=\"helm-install-tsg-traffic-engine-numa-0-8xl8g\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"247dfe7f-63a2-4ae1-ab54-4806f10f10e3\"}" timestamp=1689546704478
Jul 17 03:32:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:52.999Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_running\", asset=\"pcap-tsgx11\", asset_id=\"115\", container=\"prometheus-server-configmap-reload\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-server-6ff784d758-nghpf\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"21f91134-966a-424e-a89e-96c924c64f07\"}" timestamp=1689546704478
Jul 17 03:32:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:53.089Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx08\", asset_id=\"65\", container=\"redis\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Sync\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546696615
Jul 17 03:32:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:53.281Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_spec_paused\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"dashboard-metrics-scraper\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546708950
Jul 17 03:32:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:53.411Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_processes\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546697904
Jul 17 03:32:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:53.528Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_net_dev_group\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"vethb1d236b8\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546708328
Jul 17 03:32:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:53.797Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_info\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", job_name=\"helm-install-ipmi-exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546708663
Jul 17 03:32:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:53.830Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"msh-tsgx08\", asset_id=\"65\", container=\"kube-multus\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"iowaiting\"}" timestamp=1689546688203
Jul 17 03:32:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:53.945Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"msh-tsgx08\", asset_id=\"65\", cpu=\"42\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546708328
Jul 17 03:32:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:53.962Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cpu=\"11\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546699409
Jul 17 03:32:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:54.013Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx08\", asset_id=\"65\", cpu=\"45\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546708328
Jul 17 03:32:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:54.031Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cpu=\"105\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546699409
Jul 17 03:32:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:54.036Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_multicast_total\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens7f1np1\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546708328
Jul 17 03:32:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:54.050Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_cpu_seconds_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-podd360d8a3_9aa3_40d2_ad94_ccc470a435ce.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"slice\"}" timestamp=1689546700884
Jul 17 03:32:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:54.055Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_annotations\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"ipmi-exporter-6f8464dd94-792kb\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"da81ea3f-d797-4414-9e38-bc69ec182d3b\"}" timestamp=1689546708663
Jul 17 03:32:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:54.107Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_usage_seconds_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", container=\"mrapm-device\", cpu=\"total\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546695191
Jul 17 03:32:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:54.127Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2dpartlabel-TSG\\\\x2dOS\\\\x2dSYSROOT.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689546700884
Jul 17 03:32:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:54.137Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_packets_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"cadvisor\", interface=\"br_dign_c\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546709420
Jul 17 03:32:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:54.178Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", container=\"dign-server-dns\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Async\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546700301
Jul 17 03:32:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:54.183Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-6d37513dcd278f8bff5cd62c743ddf13772cb8562b60bd7cb7e6a12f2bc001e4-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689546700884
Jul 17 03:32:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:54.278Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_tasks_max\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"plymouth-quit-wait.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689546700884
Jul 17 03:32:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:54.349Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_created\", asset=\"msh-tsgx08\", asset_id=\"65\", configmap=\"chart-content-mrzcpd-exporter\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546708950
Jul 17 03:32:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:54.405Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_spec_parallelism\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", job_name=\"helm-install-tsg-traffic-engine-numa-0\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546708950
Jul 17 03:32:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:54.439Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-2f31f5331283cced4d08fd87c457d2e7dc7332332f1df1b6b00b0adc4b274ef5.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"scope\"}" timestamp=1689546711179
Jul 17 03:32:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:54.525Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_terminated\", asset=\"pcap-tsgx07\", asset_id=\"85\", container=\"helm\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-systemd-exporter-4r9xs\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"a2202e36-b444-4b6f-8c3b-cb1e12ec7986\"}" timestamp=1689546702058
Jul 17 03:32:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:54.540Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_device_id\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"cnibr0\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546703391
Jul 17 03:32:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:54.559Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_max_fds\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"rsyslog.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689546711179
Jul 17 03:32:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:54.559Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_name_assign_type\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"vethcf8870d8\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546699409
Jul 17 03:32:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:54.605Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_constraint_errors_received_total\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_6\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546708328
Jul 17 03:32:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:54.623Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sysstat-summary.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"service\"}" timestamp=1689546712135
Jul 17 03:32:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:54.644Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db125\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546710108
Jul 17 03:32:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:54.802Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_packets_total\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth0748f482\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546708328
Jul 17 03:32:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:54.842Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-journal-flush.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689546712135
Jul 17 03:32:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:54.862Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"clickhouse_background_message_broker_schedule_pool_task\", asset=\"pcap-dellr740-dt003\", asset_id=\"35\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"NC-Clickhouse-pcap-dellr740-dt003\", endpoint_id=\"85\", module=\"NC-Clickhouse\", module_id=\"29\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546713231
Jul 17 03:32:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:54.943Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_device_id\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"vethcd3875c5\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546703391
Jul 17 03:32:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:54.997Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"msh-tsgx08\", asset_id=\"65\", container=\"firewall\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"cadvisor\", failure_type=\"pgfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"hierarchy\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689546697891
Jul 17 03:32:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:55.048Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx12\", asset_id=\"113\", cpu=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546703391
Jul 17 03:32:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:55.074Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx12\", asset_id=\"113\", cpu=\"35\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546703391
Jul 17 03:32:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:55.204Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"clickhouse_os_irq_time_cpu29\", asset=\"pcap-dellr740-dt003\", asset_id=\"35\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"NC-Clickhouse-pcap-dellr740-dt003\", endpoint_id=\"85\", module=\"NC-Clickhouse\", module_id=\"29\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546713231
Jul 17 03:32:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:55.313Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_netstat_Tcp_InSegs\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546703391
Jul 17 03:32:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:55.374Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"phy_tx_bits_total\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", dev=\"virtio_dign_c\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"mrzcpd-exporter\"}" timestamp=1689546704559
Jul 17 03:32:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:55.393Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cpu=\"18\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546699409
Jul 17 03:32:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:55.486Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db101\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546705184
Jul 17 03:32:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:56.955Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"45\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546705883
Jul 17 03:32:58 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:58.027Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"pcap-tsgx03\", asset_id=\"73\", condition=\"Available\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"tsg-traffic-engine-numa-0-firewall\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"unknown\"}" timestamp=1689546713389
Jul 17 03:32:58 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:58.388Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"75\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546705883
Jul 17 03:32:58 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:58.426Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_uname_info\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", domainname=\"(none)\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", machine=\"x86_64\", module=\"TSG-OS\", module_id=\"77\", nodename=\"pcap-tsgx03\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", release=\"5.17.15-1.el8.x86_64\", service=\"prometheus-node-exporter\", sysname=\"Linux\", version=\"#1 SMP PREEMPT Wed Jun 15 02:07:24 UTC 2022\"}" timestamp=1689546705883
Jul 17 03:32:58 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:58.471Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-aebe8d09\\\\x2d6d5d\\\\x2d4eda\\\\x2db90d\\\\x2d7be24e2da709-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2dqs225.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689546703372
Jul 17 03:32:58 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:58.662Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"OLAP-Node-Exporter-pcap-dellr740-dt001\", endpoint_id=\"251\", module=\"OLAP-Node-Exporter\", module_id=\"51\", name=\"32\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\", type=\"Processor\"}" timestamp=1689546717558
Jul 17 03:32:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:59.181Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_hwmon_temp_max_celsius\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", chip=\"platform_coretemp_1\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"OLAP-Node-Exporter-pcap-dellr740-dt001\", endpoint_id=\"251\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\", sensor=\"temp15\"}" timestamp=1689546717558
Jul 17 03:32:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:59.751Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_writes_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546714596
Jul 17 03:32:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:32:59.805Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"112\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546709083
Jul 17 03:33:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:01.430Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"prometheus-kube-state-metrics\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546707425
Jul 17 03:33:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:02.517Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_spec_paused\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"kubernetes-dashboard\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546711412
Jul 17 03:33:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:02.645Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx02\", asset_id=\"63\", cpu=\"6\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546718318
Jul 17 03:33:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:02.677Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_info\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_endpoint=\"traefik\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546711412
Jul 17 03:33:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:02.696Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_init_container_status_terminated\", asset=\"msh-tsgx02\", asset_id=\"63\", container=\"firewall-init\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", pod=\"tsg-traffic-engine-numa-1-firewall-76f6cf59ff-9hvcf\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"4a2303d1-fff8-4c08-a906-9813964074f7\"}" timestamp=1689546711412
Jul 17 03:33:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:02.784Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-netns-cni\\\\x2d6601a439\\\\x2d4cd2\\\\x2d4bd2\\\\x2d33b1\\\\x2de33aa9707bc0.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689546708447
Jul 17 03:33:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:02.818Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"msh-tsgx02\", asset_id=\"63\", cpu=\"15\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546718318
Jul 17 03:33:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:02.847Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"95\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546718318
Jul 17 03:33:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:02.850Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_max_fds\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"mrzcpd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689546708447
Jul 17 03:33:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:02.864Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_packets_transmitted_total\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_6\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546718318
Jul 17 03:33:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:02.896Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_created\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"kube-multus-ds-45cnz\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"7adc00b9-e646-4822-9ef0-1a15d34bda7e\"}" timestamp=1689546711412
Jul 17 03:33:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:03.128Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx02\", asset_id=\"63\", cpu=\"116\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546718318
Jul 17 03:33:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:03.145Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx02\", asset_id=\"63\", cpu=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546718318
Jul 17 03:33:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:03.212Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_mtu_bytes\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"bond0\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546718318
Jul 17 03:33:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:03.221Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_max_usage_bytes\", asset=\"msh-tsgx02\", asset_id=\"63\", container=\"firewall\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689546704739
Jul 17 03:33:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:03.312Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_working_set_bytes\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689546711988
Jul 17 03:33:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:03.330Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_persistentvolume_info\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", persistentvolume=\"pvc-510dae2f-c3b6-4fc8-8fb4-b442f2d2e24c\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", storageclass=\"local-path\"}" timestamp=1689546711412
Jul 17 03:33:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:03.335Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx02\", asset_id=\"63\", cpu=\"1\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546718318
Jul 17 03:33:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:03.413Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"msh-tsgx02\", asset_id=\"63\", cpu=\"72\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546718318
Jul 17 03:33:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:03.461Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas_updated\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"prometheus-alertmanager\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546711412
Jul 17 03:33:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:03.483Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_terminated\", asset=\"msh-tsgx02\", asset_id=\"63\", container=\"helm\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-mrzcpd-exporter-lvwd9\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"225625d7-83ea-4ba9-8029-a534c5c1fb0c\"}" timestamp=1689546711412
Jul 17 03:33:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:03.552Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Running\", pod=\"helm-install-prometheus-glw8n\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"bc80432c-841c-4177-817a-bf841f359b9d\"}" timestamp=1689546711412
Jul 17 03:33:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:03.574Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda5\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", fstype=\"ext4\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/var/lib/kubelet/pods/43d98779-0eca-4764-ab37-a5e20cb923be/volume-subpaths/config-volume/telegraf/1\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546718318
Jul 17 03:33:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:03.583Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_node_status_capacity\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", node=\"localhost\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", resource=\"hugepages_2Mi\", service=\"prometheus-kube-state-metrics\", unit=\"byte\"}" timestamp=1689546711412
Jul 17 03:33:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:03.614Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_owner\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", owner_is_controller=\"true\", owner_kind=\"Job\", owner_name=\"helm-install-configmap-reloader\", pod=\"helm-install-configmap-reloader-5tp4b\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"26219aeb-0511-4104-9986-0d95f0ea0019\"}" timestamp=1689546711412
Jul 17 03:33:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:04.633Z caller=dedupe.go:112 component=remote level=info remote_name=52db6f url=http://10.10.10.159:10090/nz-agent/prometheus/proxy/api/v1/write msg="Dropped sample for series that was not explicitly dropped via relabelling" ref=4436650
Jul 17 03:33:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:06.984Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2did-wwn\\\\x2d0x62cea7f085ed36002c25b4d2a14582f7\\\\x2dpart2.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689546713582
Jul 17 03:33:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:06.999Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-ba119771e5715607d3a42c32fca454c95ecaa6460887b791772863207b888a2f-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689546713582
Jul 17 03:33:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:07.098Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"rtkit-daemon.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"service\"}" timestamp=1689546713582
Jul 17 03:33:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:07.379Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-545662f5849c111446cfcc880bbca1889b0e74b9f3eac03e05978c67b980f467-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689546713582
Jul 17 03:33:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:07.868Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"nfsdcld.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689546713582
Jul 17 03:33:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:07.954Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_spec_replicas\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"prometheus-kube-state-metrics-6b768c7bf8\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546721227
Jul 17 03:33:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:08.110Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_fifo_total\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens1f1np1\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546723910
Jul 17 03:33:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:08.355Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_labels\", asset=\"twa-tsgx01\", asset_id=\"91\", configmap=\"kube-root-ca.crt\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-node-lease\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546721227
Jul 17 03:33:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:08.374Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cpu=\"93\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546723910
Jul 17 03:33:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:08.394Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cpu=\"67\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546723910
Jul 17 03:33:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:08.420Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cpu=\"125\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546723910
Jul 17 03:33:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:08.436Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"0\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546723910
Jul 17 03:33:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:08.437Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cpu=\"36\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546723910
Jul 17 03:33:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:08.452Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cpu=\"93\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546723910
Jul 17 03:33:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:08.794Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"cortex_blocks_meta_syncs_total\", agent_name=\"msh-kvm-nz001\", application=\"cortex\", component=\"querier\", datacenter=\"MSH\", instance=\"127.0.0.1:10093\"}" timestamp=1689546727261
Jul 17 03:33:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:08.798Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-infiniband-umad9.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689546713582
Jul 17 03:33:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:08.815Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-545662f5849c111446cfcc880bbca1889b0e74b9f3eac03e05978c67b980f467-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689546713582
Jul 17 03:33:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:08.830Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_compressed_total\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth07e82765\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546723910
Jul 17 03:33:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:08.848Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_packets_total\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"eno35np2\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546723910
Jul 17 03:33:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:08.865Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"53\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546723910
Jul 17 03:33:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:08.883Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_cache\", asset=\"twa-tsgx01\", asset_id=\"91\", container=\"metrics-server\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546709144
Jul 17 03:33:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:08.917Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cpu=\"65\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546723910
Jul 17 03:33:09 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:09.326Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_NameNode_CorruptBlocks\", asset=\"twa-dellr740-dt002\", asset_id=\"39\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-Hadoop-twa-dellr740-dt002-NameNode\", endpoint_id=\"337\", module=\"DC-Hadoop\", module_id=\"53\", name=\"FSNamesystem\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546727243
Jul 17 03:33:09 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:09.737Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_filteredReadRequestCount\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"babc1946f31faf5b4151981143c005cf\", table=\"default\", type=\"data\"}" timestamp=1689546728805
Jul 17 03:33:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:10.145Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_storeFileSize\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"troubleshooting_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"d2114d26e64525f0229a1f51892d452d\", table=\"index_partfile_troubleshooting_hos_bucket\", type=\"index\"}" timestamp=1689546728805
Jul 17 03:33:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:10.163Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_mobScanCellsCount\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Server\"}" timestamp=1689546728805
Jul 17 03:33:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:10.244Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_getCount\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"rtp_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"9288256a257baf160fee600e192bb7ba\", table=\"index_filename_rtp_hos_bucket\", type=\"index\"}" timestamp=1689546728805
Jul 17 03:33:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:10.291Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_compactionsQueuedCount\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"assessment_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"8795d8fa9b6a209d701175f28e5b7624\", table=\"index_filename_assessment_hos_bucket\", type=\"index\"}" timestamp=1689546728805
Jul 17 03:33:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:10.393Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_readRequestCount\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"troubleshooting_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"85895fbc8b61b2aea3a479507705491f\", table=\"index_filename_troubleshooting_hos_bucket\", type=\"index\"}" timestamp=1689546728805
Jul 17 03:33:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:10.900Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_getCount\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"firewall_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"700a07cb70ff9f2c61808c3e4df6599d\", table=\"index_filename_firewall_hos_bucket\", type=\"index\"}" timestamp=1689546728805
Jul 17 03:33:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:10.917Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_readRequestCount\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"1cbedba153d7ee8020c2ea38662055b2\", table=\"rtp_hos_bucket\", type=\"data\"}" timestamp=1689546728805
Jul 17 03:33:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:10.941Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_lastMajorCompactionAge\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"851c6c4ec37592321de395da60c856c6\", table=\"session_record_hos_bucket\", type=\"data\"}" timestamp=1689546728805
Jul 17 03:33:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:10.954Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_avgStoreFileAge\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"4c0751b93f94543630c24fe57e08484b\", table=\"troubleshooting_hos_bucket\", type=\"data\"}" timestamp=1689546728805
Jul 17 03:33:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:11.441Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_194_1_metric_majorCompactionInputSize_99_9th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546730148
Jul 17 03:33:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:11.462Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_193_1_metric_getTime_max\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546730148
Jul 17 03:33:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:11.513Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_compactedCellsCount\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Server\"}" timestamp=1689546728805
Jul 17 03:33:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:11.534Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_default_metric_scanSize_max\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546730148
Jul 17 03:33:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:11.546Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_STATS_metric_getTime_num_ops\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546728805
Jul 17 03:33:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:11.793Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_numReferenceFiles\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"session_record_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"b294aae0b11a6abdccfdd6fee84f700d\", table=\"index_time_session_record_hos_bucket\", type=\"index\"}" timestamp=1689546730148
Jul 17 03:33:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:11.846Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_tsg_galaxy_table_gtpc_knowledge_base_metric_scanTime_max\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546730148
Jul 17 03:33:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:11.874Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_default_metric_flushOutputSize_98th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546730148
Jul 17 03:33:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:11.963Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_MajorCompactionInputSize_25th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Server\"}" timestamp=1689546730148
Jul 17 03:33:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:12.135Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_session_record_hos_bucket_metric_tableWriteQueryPerSecond_15min_rate\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546730148
Jul 17 03:33:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:12.208Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_194_1_metric_putTime_median\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546730148
Jul 17 03:33:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:12.246Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_DuplicateKeyCheckTime_90th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"PhoenixIndexer\"}" timestamp=1689546730148
Jul 17 03:33:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:12.267Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Sink_ageOfLastAppliedOp_25th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Replication\"}" timestamp=1689546730148
Jul 17 03:33:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:12.366Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_194_1_metric_putTime_75th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546730148
Jul 17 03:33:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:12.377Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_193_1_metric_incrementTime_25th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546730148
Jul 17 03:33:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:12.390Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_MUTEX_metric_scanSize_25th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546730148
Jul 17 03:33:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:12.518Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_session_record_hos_bucket_metric_scanTime_num_ops\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546730148
Jul 17 03:33:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:12.533Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_4_metric_scanTime_num_ops\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546730148
Jul 17 03:33:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:12.562Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_maxFlushQueueSize\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"knowledge_base_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"13a4e6025bf464f6e219ed4362f86f50\", table=\"index_time_knowledge_base_hos_bucket\", type=\"index\"}" timestamp=1689546730148
Jul 17 03:33:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:12.576Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_2_metric_deleteTime_75th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546730148
Jul 17 03:33:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:12.592Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_numReferenceFiles\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"e62b91d4f4dda56259135f42dc231490\", table=\"firewall_hos_bucket\", type=\"data\"}" timestamp=1689546730148
Jul 17 03:33:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:12.653Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_minStoreFileAge\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"rtp_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"b25bc8b7b58f2fe2c8ddaad6e44f85fc\", table=\"index_filename_rtp_hos_bucket\", type=\"index\"}" timestamp=1689546730148
Jul 17 03:33:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:12.725Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_STATS_metric_putBatchTime_95th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546730148
Jul 17 03:33:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:12.830Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_avgStoreFileAge\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"2684b3aacb94f4f93927a3f73c0d6de6\", table=\"assessment_hos_bucket\", type=\"data\"}" timestamp=1689546730148
Jul 17 03:33:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:15.672Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx07\", asset_id=\"103\", container=\"dign-client\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Sync\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546727165
Jul 17 03:33:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:15.696Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_up_changes_total\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth507b7852\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546721230
Jul 17 03:33:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:15.736Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_ulimits_soft\", asset=\"twa-tsgx07\", asset_id=\"103\", container=\"telegraf\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", ulimit=\"max_open_files\"}" timestamp=1689546727482
Jul 17 03:33:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:15.775Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db76\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546726027
Jul 17 03:33:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:15.936Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_timex_offset_seconds\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546721230
Jul 17 03:33:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:16.216Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_packets_received_total\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_1\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546721230
Jul 17 03:33:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:16.467Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_packets_dropped_total\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"cadvisor\", interface=\"eno1\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546724450
Jul 17 03:33:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:16.484Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"26\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546721230
Jul 17 03:33:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:16.499Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"79\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546721230
Jul 17 03:33:18 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:18.516Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"clickhouse_network_receive_bytes_em2\", asset=\"twa-dellr740-dt003\", asset_id=\"41\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"NC-Clickhouse-twa-dellr740-dt003\", endpoint_id=\"91\", module=\"NC-Clickhouse\", module_id=\"29\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546738303
Jul 17 03:33:18 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:18.615Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_packets_total\", asset=\"msh-tsgx04\", asset_id=\"57\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"vethac30e70a\", endpoint=\"TSG-OS-msh-tsgx04\", endpoint_id=\"425\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546724557
Jul 17 03:33:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:19.940Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_labels\", asset=\"twa-tsgx06\", asset_id=\"101\", configmap=\"multus-cni-config\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546733949
Jul 17 03:33:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:19.975Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_restarts_total\", asset=\"twa-tsgx06\", asset_id=\"101\", container=\"configmap-reloader\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"configmap-reloader-6c55bbc49f-f76lp\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"c7f82753-6cf1-4af2-b179-5ea64140eb6d\"}" timestamp=1689546733949
Jul 17 03:33:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:20.091Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_spec_parallelism\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", job_name=\"helm-install-mrzcpd-exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546733949
Jul 17 03:33:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:20.135Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_last_seen\", asset=\"twa-tsgx06\", asset_id=\"101\", container=\"lb-tcp-80\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546732928
Jul 17 03:33:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:20.150Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_daemonset_status_number_ready\", asset=\"twa-tsgx06\", asset_id=\"101\", daemonset=\"kube-multus-ds\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546733949
Jul 17 03:33:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:20.220Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"mdmonitor.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"service\"}" timestamp=1689546724356
Jul 17 03:33:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:20.295Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-b9ed569c7da51837144927a3eea70259ac129aa89ffd5e8e96518bf89314ca06-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689546724356
Jul 17 03:33:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:20.311Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-virtual-net-vethdd9c27e1.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689546724356
Jul 17 03:33:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:20.368Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_packets_received_total\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546737612
Jul 17 03:33:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:20.531Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_bytes_total\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"cadvisor\", interface=\"ens8f1np1\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546732541
Jul 17 03:33:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:20.559Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_heap_objects\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546734840
Jul 17 03:33:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:20.565Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_ips\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", ip=\"10.42.0.67\", ip_family=\"4\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-traefik-crd-6hp8h\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"e4381804-2831-4c31-a46e-78f802625973\"}" timestamp=1689546733949
Jul 17 03:33:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:20.589Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled_time\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", pod=\"tsg-traffic-engine-numa-1-firewall-5d7bfbbbfb-4wgxr\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"ab53e5a7-f22e-498d-b5af-535ae7359350\"}" timestamp=1689546733828
Jul 17 03:33:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:20.623Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"46\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546728263
Jul 17 03:33:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:20.641Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_metadata_generation\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"dashboard-metrics-scraper\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546733828
Jul 17 03:33:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:20.667Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_packets_total\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethe9a89d59\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546737612
Jul 17 03:33:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:20.668Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"74\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546728263
Jul 17 03:33:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:20.719Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"86\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546728263
Jul 17 03:33:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:21.118Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_hwmon_temp_celsius\", asset=\"twa-tsgx06\", asset_id=\"101\", chip=\"0000:80:01_2_0000:82:00_0\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", sensor=\"temp1\", service=\"prometheus-node-exporter\"}" timestamp=1689546737612
Jul 17 03:33:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:21.264Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_labels\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-headless\", uid=\"6dca2e0d-3851-4038-848f-e692b7e8752a\"}" timestamp=1689546733828
Jul 17 03:33:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:21.352Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_name_assign_type\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"vethdf3e6cb0\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546728263
Jul 17 03:33:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:21.352Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"phy_tx_packets_total\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", dev=\"virtio_dign_c\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"mrzcpd-exporter\"}" timestamp=1689546737986
Jul 17 03:33:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:21.407Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dracut-pre-mount.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"service\"}" timestamp=1689546724356
Jul 17 03:33:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:21.435Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_sockets\", asset=\"twa-tsgx06\", asset_id=\"101\", container=\"prometheus-pushgateway\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546728572
Jul 17 03:33:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:21.442Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_commands_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cmd=\"exec\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546734840
Jul 17 03:33:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:21.771Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_entropy_available_bits\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546728263
Jul 17 03:33:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:22.101Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_gc_sys_bytes\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"OLAP-Node-Exporter-twa-dellr740-dt001\", endpoint_id=\"257\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546741405
Jul 17 03:33:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:22.132Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"67\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546728263
Jul 17 03:33:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:22.147Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"79\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546728263
Jul 17 03:33:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:22.171Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"shm\", endpoint=\"OLAP-Node-Exporter-twa-dellr740-dt001\", endpoint_id=\"257\", fstype=\"tmpfs\", module=\"OLAP-Node-Exporter\", module_id=\"51\", mountpoint=\"/data/tsg/olap/docker/containers/a942066a1dfe0197058d29b18047c3fedc9551fd6c4ca5339a135503ed438167/mounts/shm\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546741405
Jul 17 03:33:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:22.304Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_gc_duration_seconds\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"OLAP-Node-Exporter-twa-dellr740-dt001\", endpoint_id=\"257\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\", quantile=\"0.75\"}" timestamp=1689546741405
Jul 17 03:33:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:23.513Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_write_seconds_total\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda2\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546738175
Jul 17 03:33:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:23.532Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_start_time_seconds\", asset=\"msh-tsgx09\", asset_id=\"55\", container=\"prometheus-pushgateway\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546738586
Jul 17 03:33:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:23.550Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_id\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens7f1np1\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546734790
Jul 17 03:33:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:23.566Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-4c92ea3a\\\\x2d4155\\\\x2d49d3\\\\x2d9d36\\\\x2d59027e09d717-volume\\\\x2dsubpaths-config\\\\x2dpath-dign\\\\x2dserver\\\\x2ddns-1.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689546738416
Jul 17 03:33:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:23.635Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx09\", asset_id=\"55\", container=\"configmap-reloader\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Async\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546736500
Jul 17 03:33:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:23.736Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_resident_memory_bytes\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"rpcbind.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689546738416
Jul 17 03:33:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:23.881Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_annotations\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"coredns-7cbc7ddf87-8lntt\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"bfb92468-68e3-414d-a511-fe5cfafd4a5a\"}" timestamp=1689546740068
Jul 17 03:33:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:24.544Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-tmpfiles-clean.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689546731102
Jul 17 03:33:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:24.574Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"var-lib-kubelet-pods-a27a1ebc\\\\x2d016c\\\\x2d4998\\\\x2da3f3\\\\x2d85e865dfcd16-volume\\\\x2dsubpaths-config\\\\x2dvolume-telegraf-2.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689546731102
Jul 17 03:33:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:24.649Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_rdb_last_cow_size_bytes\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546741795
Jul 17 03:33:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:24.726Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-8faffc72\\\\x2d4ba1\\\\x2d4272\\\\x2d9289\\\\x2d0077b5e77c0a-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2dh9dld.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689546738416
Jul 17 03:33:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:24.769Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_annotations\", asset=\"pcap-tsgx12\", asset_id=\"117\", configmap=\"cm-policy-local-cache-redis-scripts\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546731122
Jul 17 03:33:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:24.812Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_info\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.mrzcpd-exporter.v1\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546731122
Jul 17 03:33:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:24.817Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_multicast_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"lo\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546740160
Jul 17 03:33:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:24.851Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_info\", address=\"82:11:c4:95:0e:8d\", asset=\"pcap-tsgx12\", asset_id=\"117\", broadcast=\"ff:ff:ff:ff:ff:ff\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"vethd1369752\", duplex=\"full\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operstate=\"up\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546740160
Jul 17 03:33:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:24.876Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_errs_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens1f1np1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546740160
Jul 17 03:33:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:24.899Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx09\", asset_id=\"55\", container=\"dashboard-metrics-scraper\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Async\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546730798
Jul 17 03:33:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:24.901Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_packets_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens4f1np1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546740160
Jul 17 03:33:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:24.907Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_compressed_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"vethfc5d9f9d\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546740160
Jul 17 03:33:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:24.942Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failcnt\", asset=\"msh-tsgx09\", asset_id=\"55\", container=\"exporter-merger\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546720225
Jul 17 03:33:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:25.179Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_annotations\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"mrzcpd-exporter\", uid=\"dfc53202-8aee-46de-83cb-304b87e00648\"}" timestamp=1689546731122
Jul 17 03:33:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:25.185Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-sysctl.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"oneshot\", type=\"service\"}" timestamp=1689546738416
Jul 17 03:33:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:25.205Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_memory_Inactive_bytes\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546740160
Jul 17 03:33:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:25.227Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_client_recent_max_output_buffer_bytes\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546741795
Jul 17 03:33:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:25.263Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"microcode.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"service\"}" timestamp=1689546731102
Jul 17 03:33:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:25.307Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_usage_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"coredns\", cpu=\"total\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546732769
Jul 17 03:33:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:25.314Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"nginx_vts_upstream_request_seconds_total\", asset=\"msh-dellr740-olap001\", asset_id=\"11\", backend=\"192.168.20.221:8183\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"Galaxy-gateway-nginx-msh-dellr740-olap001\", endpoint_id=\"43\", module=\"Galaxy-gateway-nginx\", module_id=\"11\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", upstream=\"qgwService\"}" timestamp=1689546744444
Jul 17 03:33:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:25.325Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"87\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546740160
Jul 17 03:33:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:25.371Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"nis-domainname.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"service\"}" timestamp=1689546731102
Jul 17 03:33:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:25.372Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-virtual-net-vethc02b7391.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689546731102
Jul 17 03:33:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:25.383Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-3ebda57eb13da2902e1c29c9ff712b1a13e50657d0aefe6ef46953cd060cc0f3-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689546731102
Jul 17 03:33:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:25.412Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx09\", asset_id=\"55\", cpu=\"95\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546734790
Jul 17 03:33:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:25.413Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"exporter-merger\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Write\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546720164
Jul 17 03:33:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:25.612Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_threads_max\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"firewall\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689546728105
Jul 17 03:33:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:25.657Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db137\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546741795
Jul 17 03:33:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:25.793Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_system_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"redis\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546736854
Jul 17 03:33:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:25.834Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_ulimits_soft\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"firewall\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", ulimit=\"max_open_files\"}" timestamp=1689546743131
Jul 17 03:33:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:25.834Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_load_average_10s\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"traefik\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546733528
Jul 17 03:33:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:25.856Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_errors_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", interface=\"virtio_dign_c\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546742694
Jul 17 03:33:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:25.899Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-92f18de24a3cebecd0ab3ab6de0c2e060025bcb4d27a3c787c096f2acb826bd1-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689546731102
Jul 17 03:33:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:26.108Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_cpu_period\", asset=\"twa-tsgx09\", asset_id=\"107\", container=\"prometheus-server\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546745490
Jul 17 03:33:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:26.124Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_mspan_inuse_bytes\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546735810
Jul 17 03:33:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:26.400Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"NetworkManager.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"service\"}" timestamp=1689546733907
Jul 17 03:33:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:26.451Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db112\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546743549
Jul 17 03:33:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:26.452Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_init_container_info\", asset=\"twa-tsgx09\", asset_id=\"107\", container=\"install-multus-binary\", container_id=\"containerd://fbad613f5397895c0b7ba2006102252d8fa3b8db8ef807676387e100a01c508e\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", image=\"ghcr.io/k8snetworkplumbingwg/multus-cni:stable\", image_id=\"sha256:6df93b9f982bcb3b92dfbb9f5c4c72cb401be6182cda6d4de41234431a51e583\", image_spec=\"ghcr.io/k8snetworkplumbingwg/multus-cni:stable\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"kube-multus-ds-bvcvj\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"680c59bb-c7ab-427d-b652-0e744860b375\"}" timestamp=1689546738744
Jul 17 03:33:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:26.461Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_namespace_created\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"consul\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546738744
Jul 17 03:33:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:26.496Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_stack_sys_bytes\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"ipmi-exporter\"}" timestamp=1689546741597
Jul 17 03:33:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:26.503Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_net_dev_group\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"lo\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546735810
Jul 17 03:33:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:26.513Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"75\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546735810
Jul 17 03:33:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:26.543Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_mapped_file\", asset=\"twa-tsgx09\", asset_id=\"107\", container=\"configmap-reloader\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546737589
Jul 17 03:33:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:26.558Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"snmptrapd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"notify\", type=\"service\"}" timestamp=1689546733907
Jul 17 03:33:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:26.646Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx09\", asset_id=\"107\", container=\"firewall\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Async\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689546736608
Jul 17 03:33:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:26.662Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-pod880aa12b_51b0_4413_88cb_d06b2fc6d0c3.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"slice\"}" timestamp=1689546733907
Jul 17 03:33:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:26.667Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:80-0000:80:01.2-0000:82:00.1-net-ens4f1np1.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689546733907
Jul 17 03:33:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:26.721Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_created\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_endpoint=\"kubernetes-dashboard\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546738744
Jul 17 03:33:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:26.741Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"tracefs\", name=\"sys-kernel-debug-tracing.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689546733907
Jul 17 03:33:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:26.786Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"pcap-kvm-cm001\", asset_id=\"45\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db224\", endpoint=\"CM-Branch-Redis-pcap-kvm-cm001\", endpoint_id=\"321\", host=\"192.168.10.157\", module=\"CM-Branch-Redis\", module_id=\"73\", nz_agent_id=\"3\", parent_asset=\"pcap-dellr740-pve001\", parent_asset_id=\"43\", project=\"Central-Management\", project_id=\"5\"}" timestamp=1689546745366
Jul 17 03:33:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:26.915Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_ulimits_soft\", asset=\"twa-tsgx09\", asset_id=\"107\", container=\"lb-tcp-80\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", ulimit=\"max_open_files\"}" timestamp=1689546742546
Jul 17 03:33:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:26.922Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_type\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.traefik-crd.v1\", service=\"prometheus-kube-state-metrics\", type=\"helm.sh/release.v1\"}" timestamp=1689546738744
Jul 17 03:33:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:26.936Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_annotations\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.tsg-traffic-engine-numa-0.v3\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546738744
Jul 17 03:33:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:26.945Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"47\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546735810
Jul 17 03:33:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:26.952Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546735810
Jul 17 03:33:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:26.963Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_ulimits_soft\", asset=\"twa-tsgx09\", asset_id=\"107\", container=\"dign-server-dns\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", ulimit=\"max_open_files\"}" timestamp=1689546741732
Jul 17 03:33:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:27.077Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-ens3f1np1.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689546733907
Jul 17 03:33:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:27.082Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"34\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546735810
Jul 17 03:33:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:27.362Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_sockstat_UDPLITE6_inuse\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546735810
Jul 17 03:33:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:27.391Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_mspan_inuse_bytes\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546743549
Jul 17 03:33:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:27.429Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"111\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546735810
Jul 17 03:33:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:27.842Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx09\", asset_id=\"107\", container=\"prometheus-server\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Async\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546736145
Jul 17 03:33:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:27.957Z caller=dedupe.go:112 component=remote level=error remote_name=52db6f url=http://10.10.10.159:10090/nz-agent/prometheus/proxy/api/v1/write msg="non-recoverable error" count=1000 exemplarCount=0 err="server returned HTTP status 400 : out of order sample"
Jul 17 03:33:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:28.020Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_reads_bytes_total\", asset=\"twa-tsgx09\", asset_id=\"107\", container=\"kube-multus\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546740101
Jul 17 03:33:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:28.376Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"22\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546742882
Jul 17 03:33:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:28.404Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_mod_info\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"github.com/prometheus/client_model\", nz_agent_id=\"3\", program=\"systemd_exporter\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", version=\"v0.2.0\"}" timestamp=1689546734114
Jul 17 03:33:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:28.420Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"clickhouse_network_send_drop_bond0\", asset=\"msh-dellr740-dt003\", asset_id=\"9\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Clickhouse-msh-dellr740-dt003\", endpoint_id=\"75\", module=\"NC-Clickhouse\", module_id=\"29\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546746493
Jul 17 03:33:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:28.425Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"msh-tsgx10\", asset_id=\"69\", condition=\"Available\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"traefik\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"unknown\"}" timestamp=1689546735682
Jul 17 03:33:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:28.545Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"ftx_missed_total\", app=\"sapp4-0\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"nf_0_fw\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"mrzcpd-exporter\", serviceFunctionIndex=\"0\"}" timestamp=1689546733863
Jul 17 03:33:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:28.614Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"122\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546742882
Jul 17 03:33:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:28.629Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db18\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546739446
Jul 17 03:33:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:28.643Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"local-path-provisioner\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Total\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546741413
Jul 17 03:33:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:28.657Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db247\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546739446
Jul 17 03:33:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:28.901Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"73\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546742882
Jul 17 03:33:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:28.922Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_annotations\", asset=\"msh-tsgx10\", asset_id=\"69\", configmap=\"chart-content-cm-policy-local-cache\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546735682
Jul 17 03:33:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:28.943Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_init_container_status_restarts_total\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"tsg-dign-init\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"dign-server-pf96k\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"75dc4160-da0a-4767-823a-227b8dd303c1\"}" timestamp=1689546735682
Jul 17 03:33:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:28.958Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"lb-tcp-443\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"running\"}" timestamp=1689546739507
Jul 17 03:33:29 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:29.114Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas_ready\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"configmap-reloader\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546735682
Jul 17 03:33:29 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:29.131Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_labels\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_endpoint=\"prometheus-alertmanager\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546735682
Jul 17 03:33:29 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:29.145Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_last_gc_time_seconds\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546739446
Jul 17 03:33:29 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:29.161Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_created\", asset=\"msh-tsgx10\", asset_id=\"69\", configmap=\"kube-root-ca.crt\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546735682
Jul 17 03:33:30 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:30.065Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_protocol_type\", asset=\"pcap-tsgx02\", asset_id=\"75\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"eno36np3\", endpoint=\"TSG-OS-pcap-tsgx02\", endpoint_id=\"469\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546736442
Jul 17 03:33:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:31.018Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_frame_total\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens4f3np3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546740939
Jul 17 03:33:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:31.528Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"16\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546739093
Jul 17 03:33:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:31.554Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Tcp_Pkt\", app_name=\"sapp4\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", vsysId=\"1\"}" timestamp=1689546742297
Jul 17 03:33:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:31.790Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_cache\", asset=\"msh-tsgx05\", asset_id=\"67\", container=\"dign-server-dns\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546727414
Jul 17 03:33:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:31.807Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_errors_total\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"cadvisor\", interface=\"virtio_dign_c\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546734166
Jul 17 03:33:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:31.882Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:00-0000:00:01.1-0000:01:00.0-infiniband-mlx5_0.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689546746907
Jul 17 03:33:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:32.138Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_drop_total\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens4f1np1\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546740791
Jul 17 03:33:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:32.173Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-df0a65bb34ddb9c67c5a1c08ef9fcd3449025d6dd44fdee7a4db87d244abedab.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"scope\"}" timestamp=1689546742644
Jul 17 03:33:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:32.185Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_packets_total\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"eno34np1\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546740939
Jul 17 03:33:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:32.228Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"25\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546740939
Jul 17 03:33:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:32.236Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"prometheus-pushgateway\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546750436
Jul 17 03:33:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:32.262Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_type\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"chart-values-traefik-crd\", service=\"prometheus-kube-state-metrics\", type=\"Opaque\"}" timestamp=1689546750436
Jul 17 03:33:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:32.343Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"57\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546740939
Jul 17 03:33:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:32.412Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"16\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546740939
Jul 17 03:33:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:32.431Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_info\", asset=\"twa-tsgx03\", asset_id=\"95\", created_by_kind=\"Job\", created_by_name=\"helm-install-mrzcpd-exporter\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", host_ip=\"10.10.30.163\", host_network=\"false\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", node=\"localhost\", nz_agent_id=\"3\", pod=\"helm-install-mrzcpd-exporter-f8z4r\", pod_ip=\"10.42.0.90\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"87c13380-1ef8-41c4-9283-bdd9bd81298d\"}" timestamp=1689546750436
Jul 17 03:33:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:32.435Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_write_seconds_total\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/run\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546729939
Jul 17 03:33:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:32.451Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failcnt\", asset=\"msh-tsgx05\", asset_id=\"67\", container=\"firewall\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689546720139
Jul 17 03:33:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:32.465Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth39378f16\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546739093
Jul 17 03:33:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:32.492Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_annotations\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-0-firewall-69747b4bf9\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546750436
Jul 17 03:33:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:32.499Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_bytes_total\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"eno35np2\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546739093
Jul 17 03:33:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:32.512Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.grpc.v1.cri-sandboxes-c36bbd943b007854126bb5e121d9774579cbfee4d4666edddd5c6b1038f9db35-shm.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689546747691
Jul 17 03:33:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:32.533Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"19\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546739093
Jul 17 03:33:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:32.543Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_ports\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_endpoint=\"ipmi-exporter\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", port_name=\"http\", port_number=\"9290\", port_protocol=\"TCP\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546739276
Jul 17 03:33:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:32.561Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2did-wwn\\\\x2d0x62cea7f0848285002c25c6341fb95721.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689546746907
Jul 17 03:33:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:32.565Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_daemonset_created\", asset=\"twa-tsgx05\", asset_id=\"99\", daemonset=\"kube-multus-ds\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546739894
Jul 17 03:33:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:32.566Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_swap_limit_bytes\", asset=\"twa-tsgx05\", asset_id=\"99\", container=\"prometheus-node-exporter\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546738535
Jul 17 03:33:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:32.582Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_start_time\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-tsg-traffic-engine-numa-1-lv7qp\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"aa621a66-69cd-4fef-a816-ab5978f42426\"}" timestamp=1689546744068
Jul 17 03:33:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:32.584Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_observed_generation\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"traefik-64f55bb67d\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546739894
Jul 17 03:33:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:32.595Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"twa-tsgx05\", asset_id=\"99\", container=\"prometheus-server-configmap-reload\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"running\"}" timestamp=1689546732744
Jul 17 03:33:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:32.602Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"91\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546740939
Jul 17 03:33:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:32.615Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_state_id\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_1\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546739093
Jul 17 03:33:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:32.621Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"emergency.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"idle\", type=\"service\"}" timestamp=1689546746907
Jul 17 03:33:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:32.628Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_timer_last_trigger_seconds\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sysstat-collect.timer\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689546747691
Jul 17 03:33:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:32.649Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_readonly\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", fstype=\"ext4\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/data\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546740605
Jul 17 03:33:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:32.690Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_metadata_resource_version\", asset=\"twa-tsgx03\", asset_id=\"95\", configmap=\"shaping-tsg-traffic-engine-numa-0\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546750436
Jul 17 03:33:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:32.695Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_mtu_bytes\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens6f0np0\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546740605
Jul 17 03:33:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:32.704Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-pod59e0f471_ccc6_4ff5_8571_7fb136f39b65.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"slice\"}" timestamp=1689546742644
Jul 17 03:33:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:32.722Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_compressed_total\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethc3a959fd\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546740605
Jul 17 03:33:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:32.723Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_annotations\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"chart-values-tsg-traffic-engine-numa-1\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546744068
Jul 17 03:33:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:32.729Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-netns-cni\\\\x2d0061f8fb\\\\x2dd217\\\\x2d850b\\\\x2df649\\\\x2d91dc10c72148.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689546747691
Jul 17 03:33:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:32.793Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_multicast_total\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth39378f16\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546739093
Jul 17 03:33:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:32.803Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_usage_bytes\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"metrics-server\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546727751
Jul 17 03:33:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:32.815Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"gssproxy.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"service\"}" timestamp=1689546742644
Jul 17 03:33:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:32.842Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_carrier_total\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens6f0np0\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546739093
Jul 17 03:33:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:32.885Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_annotations\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_endpoint=\"prometheus-pushgateway\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546739276
Jul 17 03:33:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:32.892Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-cm-policy-local-cache-tmf7f\", project=\"TSG-OS\", project_id=\"7\", reason=\"Evicted\", service=\"prometheus-kube-state-metrics\", uid=\"126aa538-e062-49ba-b634-423433ca33f0\"}" timestamp=1689546750436
Jul 17 03:33:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:32.913Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_bytes_total\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"eno33np0\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546739093
Jul 17 03:33:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:32.996Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-virtual-net-veth3448a07b.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689546742644
Jul 17 03:33:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:33.128Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_bytes_total\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth23aaa632\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546739093
Jul 17 03:33:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:33.158Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-841e23492d6c7cb72b6d23a21b11e15a6e93fcbc283f2b194cdc51818f13d2f2.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"scope\"}" timestamp=1689546746907
Jul 17 03:33:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:33.175Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-03b47e9e\\\\x2db9a5\\\\x2d462d\\\\x2db6ff\\\\x2d6e3865bd5555-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2ddxz55.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689546746907
Jul 17 03:33:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:33.194Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-pod1df6bd2b_0fea_4549_94c2_ddadc3bbc3e4.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"slice\"}" timestamp=1689546746907
Jul 17 03:33:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:33.267Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-02088315af30b64a8f3cd62d3166bc9b839dbf7326e16a2f6b89da58b7113ce0-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689546746907
Jul 17 03:33:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:33.289Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-7fa2f756f836049d0f9b218b6417624432ce0e3e583e92eb042cd9d40f8d53e5.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"scope\"}" timestamp=1689546747691
Jul 17 03:33:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:33.653Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.grpc.v1.cri-sandboxes-9740e9c7c884c99ca8541cb2bd9e08d24f01204bf3c4004342628ac96d47baf7-shm.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689546747691
Jul 17 03:33:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:34.034Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"30\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546740605
Jul 17 03:33:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:34.299Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_metadata_resource_version\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"chart-values-ipmi-exporter\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546739894
Jul 17 03:33:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:34.312Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_threads\", asset=\"twa-tsgx05\", asset_id=\"99\", container=\"dashboard-metrics-scraper\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546729633
Jul 17 03:33:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:34.335Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_labels\", asset=\"twa-tsgx05\", asset_id=\"99\", configmap=\"kube-root-ca.crt\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-public\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546739894
Jul 17 03:33:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:34.514Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_mapped_file\", asset=\"twa-tsgx05\", asset_id=\"99\", container=\"telegraf\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689546726414
Jul 17 03:33:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:34.540Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_created\", asset=\"twa-tsgx05\", asset_id=\"99\", configmap=\"prometheus-alertmanager\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546739894
Jul 17 03:33:36 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:36.271Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx09\", asset_id=\"83\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx09\", endpoint_id=\"477\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2did-wwn\\\\x2d0x62cea7f0848254002c2c7c7ad9e3acee\\\\x2dpart4.device\", nz_agent_id=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689546747845
Jul 17 03:33:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:40.824Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db254\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546753827
Jul 17 03:33:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:40.858Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"auditd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"service\"}" timestamp=1689546755764
Jul 17 03:33:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:40.910Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-62b305b4bfc0192b0144642dbfe3dc33884c68e5c46df9ed8876ff4d3b659b52.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"scope\"}" timestamp=1689546755764
Jul 17 03:33:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:41.246Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_memory_Committed_AS_bytes\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546751647
Jul 17 03:33:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:41.285Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_address_assign_type\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethbcbc9544\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546751647
Jul 17 03:33:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:41.382Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx04\", asset_id=\"97\", container=\"firewall\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Read\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689546748787
Jul 17 03:33:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:41.465Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_type\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.traefik.v1\", service=\"prometheus-kube-state-metrics\", type=\"helm.sh/release.v1\"}" timestamp=1689546760034
Jul 17 03:33:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:41.483Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_10_193_1_metric_scanSize_median\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546759757
Jul 17 03:33:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:41.484Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_frame_total\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth01fab232\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546751647
Jul 17 03:33:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:41.890Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-journal-catalog-update.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"oneshot\", type=\"service\"}" timestamp=1689546755764
Jul 17 03:33:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:41.996Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_info\", asset=\"twa-tsgx04\", asset_id=\"97\", board_id=\"MT_0000000359\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_7\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", firmware_version=\"22.28.4000\", hca_type=\"MT4125\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546751647
Jul 17 03:33:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:42.114Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db192\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546753827
Jul 17 03:33:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:42.167Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_memStoreSize\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"4b09a1b465bbca76e85bc306bdde8ad2\", table=\"default\", type=\"data\"}" timestamp=1689546759757
Jul 17 03:33:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:42.260Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"6\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546751647
Jul 17 03:33:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:42.380Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"irqbalance.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"service\"}" timestamp=1689546755764
Jul 17 03:33:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:42.478Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_minStoreFileAge\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"knowledge_base_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"a928f369daa6b9514ff873a7fb1d5407\", table=\"index_filename_knowledge_base_hos_bucket\", type=\"index\"}" timestamp=1689546759757
Jul 17 03:33:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:42.493Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_system_table_bucket_meta_metric_majorCompactionOutputFileCount_median\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546759757
Jul 17 03:33:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:42.507Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", cpu=\"0\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546756242
Jul 17 03:33:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:42.517Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", cpu=\"120\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546756242
Jul 17 03:33:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:42.534Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-podd20d4034_164e_4450_a851_056ee84ef97e.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"slice\"}" timestamp=1689546760671
Jul 17 03:33:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:42.545Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", container=\"coredns\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"cadvisor\", failure_type=\"pgfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"container\"}" timestamp=1689546759462
Jul 17 03:33:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:42.563Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_errs_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth54027b61\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546756242
Jul 17 03:33:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:42.580Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Dup_Udp_Pkt\", app_name=\"sapp4\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", vsysId=\"1\"}" timestamp=1689546747581
Jul 17 03:33:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:42.581Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_labels\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_endpoint=\"cm-policy-local-cache-redis-master\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546755154
Jul 17 03:33:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:42.598Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_state_started\", asset=\"pcap-tsgx01\", asset_id=\"71\", container=\"exporter-merger\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"mrzcpd-exporter-5cf7f967cc-sgsdt\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"23a4edf9-c51a-4ccf-aa70-4b82ecf894c2\"}" timestamp=1689546755154
Jul 17 03:33:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:42.605Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-4d40e67b\\\\x2dcb75\\\\x2d462e\\\\x2db6c7\\\\x2dc667d37b9d55-volume\\\\x2dsubpaths-config\\\\x2dpath-dign\\\\x2dclient-3.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689546760671
Jul 17 03:33:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:42.613Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", cpu=\"121\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546756242
Jul 17 03:33:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:42.651Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_created\", asset=\"pcap-tsgx01\", asset_id=\"71\", configmap=\"chart-content-cm-policy-local-cache\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546755154
Jul 17 03:33:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:42.668Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_scrape_collector_success\", asset=\"pcap-tsgx01\", asset_id=\"71\", collector=\"diskstats\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546756242
Jul 17 03:33:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:42.669Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:c0-0000:c0:01.1-0000:c1:00.1-infiniband_mad-umad9.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689546760671
Jul 17 03:33:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:42.707Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_labels\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"cm-policy-local-cache-redis-master\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546755154
Jul 17 03:33:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:42.725Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", cpu=\"22\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546756242
Jul 17 03:33:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:42.737Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-sda5.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689546760671
Jul 17 03:33:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:42.737Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_scaling_frequency_max_hertz\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", cpu=\"1\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Gohangout-msh-dellr740-dt002\", endpoint_id=\"167\", module=\"DC-Gohangout\", module_id=\"35\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546762624
Jul 17 03:33:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:42.803Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_name_assign_type\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth7573d597\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546756242
Jul 17 03:33:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:42.902Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"tmpfs\", name=\"data-var-lib-kubelet-pods-4d40e67b\\\\x2dcb75\\\\x2d462e\\\\x2db6c7\\\\x2dc667d37b9d55-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2dk22rg.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689546760671
Jul 17 03:33:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:42.920Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"slice\"}" timestamp=1689546760671
Jul 17 03:33:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:43.053Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-b975f5bfa7085daf7ea4cf8c6128f56b14518d193379482563f1392151c1cd72.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"scope\"}" timestamp=1689546760671
Jul 17 03:33:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:43.152Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_nf_conntrack_stat_search_restart\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546756242
Jul 17 03:33:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:43.165Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"phy_tx_packets_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", dev=\"ens1f1np1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"mrzcpd-exporter\"}" timestamp=1689546759613
Jul 17 03:33:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:43.177Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", cpu=\"17\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546756242
Jul 17 03:33:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:43.186Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", cpu=\"17\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546756242
Jul 17 03:33:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:43.272Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"tmpfs\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/data/var/lib/kubelet/pods/50480520-32d8-413b-9354-fde3e50919a7/volumes/kubernetes.io~projected/kube-api-access-sv7tl\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546756242
Jul 17 03:33:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:43.288Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_created\", asset=\"pcap-tsgx01\", asset_id=\"71\", configmap=\"kube-root-ca.crt\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-public\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546755154
Jul 17 03:33:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:43.453Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Udp_Offload_Bit\", app_name=\"sapp4\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", vsysId=\"1\"}" timestamp=1689546749178
Jul 17 03:33:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:43.525Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_file_descriptors\", asset=\"pcap-tsgx01\", asset_id=\"71\", container=\"telegraf-security\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689546756046
Jul 17 03:33:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:43.544Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-d57d31b4\\\\x2d5150\\\\x2d4e58\\\\x2d8d32\\\\x2d5e87c6e75897-volume\\\\x2dsubpaths-config\\\\x2dpath-dign\\\\x2dserver\\\\x2ddns-1.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689546760671
Jul 17 03:33:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:43.663Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", cpu=\"43\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546756242
Jul 17 03:33:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:43.745Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", container=\"prometheus-server-configmap-reload\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"cadvisor\", failure_type=\"pgfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"container\"}" timestamp=1689546756308
Jul 17 03:33:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:43.901Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-28e1a3f06527252e3664ac7f65e262677a31fe53577da021002e6ae46b5ead70-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689546760671
Jul 17 03:33:46 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:46.804Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_LoginFailureNumOps\", asset=\"pcap-dellr740-dt003\", asset_id=\"35\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt003-Hmaster\", endpoint_id=\"387\", module=\"DC-HBase\", module_id=\"3\", name=\"UgiMetrics\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546766370
Jul 17 03:33:46 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:46.843Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_SnapshotObserverSnapshotFetchTime_99_9th_percentile\", asset=\"pcap-dellr740-dt003\", asset_id=\"35\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt003-Hmaster\", endpoint_id=\"387\", module=\"DC-HBase\", module_id=\"3\", name=\"Master\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Quotas\"}" timestamp=1689546766370
Jul 17 03:33:48 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:48.874Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_maxStoreFileAge\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"session_record_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"5f0899b44135b34982bf8c9cfd51c7b4\", table=\"index_time_session_record_hos_bucket\", type=\"index\"}" timestamp=1689546767438
Jul 17 03:33:48 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:48.919Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_ScanTime_99th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Server\"}" timestamp=1689546767438
Jul 17 03:33:48 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:48.929Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_readRequestCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"rtp_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"19bdc5505a765d5acc400a971653a326\", table=\"index_time_rtp_hos_bucket\", type=\"index\"}" timestamp=1689546767438
Jul 17 03:33:48 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:48.956Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_193_1_metric_deleteBatchTime_75th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546767438
Jul 17 03:33:48 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:48.976Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_194_4_metric_compactionOutputSize_min\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546767438
Jul 17 03:33:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:49.105Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_3_metric_compactionTime_99_9th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546767438
Jul 17 03:33:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:49.201Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_1_metric_compactionInputFileCount_75th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546767438
Jul 17 03:33:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:49.274Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_scanCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"session_record_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"88a9bbe2136852f7ad790e6c839f28d2\", table=\"index_time_session_record_hos_bucket\", type=\"index\"}" timestamp=1689546767438
Jul 17 03:33:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:49.497Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_194_1_metric_incrementTime_90th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546767438
Jul 17 03:33:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:49.670Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"twa-tsgx10\", asset_id=\"109\", condition=\"Progressing\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"coredns\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"true\"}" timestamp=1689546759668
Jul 17 03:33:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:49.980Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_frequency_min_hertz\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"125\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546766709
Jul 17 03:33:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:50.009Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db74\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546767556
Jul 17 03:33:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:50.015Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_spec_type\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-master\", type=\"NodePort\", uid=\"a8b7c5af-aad2-4b80-8942-f54cbe43a552\"}" timestamp=1689546759668
Jul 17 03:33:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:50.033Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-infiniband-umad10.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689546766554
Jul 17 03:33:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:50.059Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"initrd.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"target\"}" timestamp=1689546766554
Jul 17 03:33:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:50.135Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_writes_merged_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/1d0a9283baaeb68215db2c28cfe83a4ec2014931320cc1ec38d111b6cd89d7b3/shm\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546763571
Jul 17 03:33:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:50.212Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_errs_total\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens4f1np1\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546761797
Jul 17 03:33:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:50.351Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens4f2np2\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546766709
Jul 17 03:33:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:50.369Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"cm-policy-local-cache-redis-master-69967b5879-xfrv9\", project=\"TSG-OS\", project_id=\"7\", reason=\"UnexpectedAdmissionError\", service=\"prometheus-kube-state-metrics\", uid=\"a2024985-d999-4ae2-847f-a93fc2842f87\"}" timestamp=1689546759668
Jul 17 03:33:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:50.387Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_multicast_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth8b4b4595\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546766709
Jul 17 03:33:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:50.395Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx10\", asset_id=\"109\", cpu=\"5\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546761797
Jul 17 03:33:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:50.413Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"119\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546766709
Jul 17 03:33:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:50.444Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_io_time_weighted_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/var/lib/kubelet/pods/1617bf89-68ca-42e6-8a6e-a79f76bf18ce/volumes/kubernetes.io~projected/kube-api-access-hjpfx\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546763571
Jul 17 03:33:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:50.511Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_labels\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-0-firewall-7cbb5bf99c\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546761568
Jul 17 03:33:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:50.581Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_owner\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", owner_is_controller=\"true\", owner_kind=\"ReplicaSet\", owner_name=\"prometheus-alertmanager-6df7f457b6\", pod=\"prometheus-alertmanager-6df7f457b6-ss2vn\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"9d049c99-7658-4134-a1b4-cb83e78cc0a3\"}" timestamp=1689546761568
Jul 17 03:33:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:50.592Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_packets_dropped_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", interface=\"ens4f0np0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546767463
Jul 17 03:33:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:50.607Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_sector_writes_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"overlay_0-177\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546763571
Jul 17 03:33:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:50.609Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"27\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546766709
Jul 17 03:33:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:50.633Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Send_Tcp_S_A\", app_name=\"sapp4\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", vsysId=\"1\"}" timestamp=1689546768675
Jul 17 03:33:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:50.788Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_flags\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"bond0\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546766709
Jul 17 03:33:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:50.876Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db240\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546767556
Jul 17 03:33:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:50.916Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_owner\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", owner_is_controller=\"true\", owner_kind=\"Deployment\", owner_name=\"ipmi-exporter\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"ipmi-exporter-6f8464dd94\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546761568
Jul 17 03:33:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:50.930Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_inodes_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"overlay_0-259\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546763571
Jul 17 03:33:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:51.050Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-udev-trigger.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689546766554
Jul 17 03:33:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:51.211Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"86\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546766709
Jul 17 03:33:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:51.330Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_DataNode_MemHeapUsedM\", asset=\"twa-dellr740-dt002\", asset_id=\"39\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-Hadoop-twa-dellr740-dt002-DataNode\", endpoint_id=\"197\", module=\"DC-Hadoop\", module_id=\"53\", name=\"JvmMetrics\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546770857
Jul 17 03:33:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:51.538Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_ulimits_soft\", asset=\"twa-tsgx08\", asset_id=\"105\", container=\"ipmi-exporter\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", ulimit=\"max_open_files\"}" timestamp=1689546758696
Jul 17 03:33:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:51.668Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_metadata_generation\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"prometheus-kube-state-metrics-6b768c7bf8\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546761568
Jul 17 03:33:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:51.688Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_unicast_packets_received_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_0\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546766709
Jul 17 03:33:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:51.704Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-sda.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689546766554
Jul 17 03:33:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:52.621Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", cpu=\"58\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546769337
Jul 17 03:33:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:52.660Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_mtu_bytes\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"eno36np3\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546769337
Jul 17 03:33:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:52.677Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_colls_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"vethec52c02a\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546769337
Jul 17 03:33:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:52.731Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"ostree-remount.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689546765496
Jul 17 03:33:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:52.750Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", phase=\"Failed\", pod=\"dign-client-gmxdm\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"4aff81f6-8efe-43da-b4c0-830d2bd4ff96\"}" timestamp=1689546764478
Jul 17 03:33:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:52.794Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-4bd025708f583c4ffee2677091b4b61981ab287dc1df42dabf020acf814abcf6.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"scope\"}" timestamp=1689546765496
Jul 17 03:33:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:52.892Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"shm\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/b2dbd13587334631c73e2030db619b01ed7a569317e4ba1e4809fa7be46a9a0a/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546769337
Jul 17 03:33:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:52.914Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", container=\"configmap-reloader\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Discard\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546761824
Jul 17 03:33:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:53.239Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"msh-tsgx08\", asset_id=\"65\", cpu=\"22\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546768328
Jul 17 03:33:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:53.452Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth4e8b65d2\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546763391
Jul 17 03:33:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:53.683Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_netstat_Icmp6_InMsgs\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546769337
Jul 17 03:33:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:53.872Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", container=\"metrics\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Read\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546766404
Jul 17 03:33:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:53.919Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-dc43bbf15c647dd6bdac70cc8b80fc4828005aafb126dca2e19defbfccc04de0.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"scope\"}" timestamp=1689546765496
Jul 17 03:33:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:53.937Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"ipmi-exporter-6f8464dd94-ghpck\", project=\"TSG-OS\", project_id=\"7\", reason=\"NodeAffinity\", service=\"prometheus-kube-state-metrics\", uid=\"ff360f8e-d8b6-4451-9e19-c901d1e7ae8c\"}" timestamp=1689546768950
Jul 17 03:33:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:53.944Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sysstat.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689546765496
Jul 17 03:33:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:53.961Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_up_changes_total\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"cnibr0\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546768328
Jul 17 03:33:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:54.149Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_scaling_frequency_max_hertz\", asset=\"pcap-dellr740-dt002\", asset_id=\"33\", cpu=\"14\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"NC-Gohangout-pcap-dellr740-dt002\", endpoint_id=\"173\", module=\"DC-Gohangout\", module_id=\"35\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546772666
Jul 17 03:33:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:54.218Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"cadvisor\", failure_type=\"pgfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"hierarchy\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689546765736
Jul 17 03:33:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:54.237Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", container=\"lb-tcp-80\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"iowaiting\"}" timestamp=1689546765142
Jul 17 03:33:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:54.244Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db96\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546770108
Jul 17 03:33:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:54.259Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"rpc-statd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689546772135
Jul 17 03:33:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:54.271Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_status_succeeded\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", job_name=\"helm-install-traefik\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546764478
Jul 17 03:33:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:54.288Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_annotations\", asset=\"pcap-tsgx07\", asset_id=\"85\", configmap=\"chart-content-prometheus\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546762058
Jul 17 03:33:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:54.316Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx12\", asset_id=\"113\", cpu=\"120\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546763391
Jul 17 03:33:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:54.354Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_size_bytes\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda5\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", fstype=\"ext4\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/data\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546768328
Jul 17 03:33:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:54.376Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Unknown\", pod=\"helm-install-tsg-traffic-engine-numa-1-f7wwg\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"06fbf17e-ade1-4a2a-aaea-eb530d9bc31e\"}" timestamp=1689546762058
Jul 17 03:33:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:54.395Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db203\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546770108
Jul 17 03:33:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:54.417Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_complete\", asset=\"msh-tsgx08\", asset_id=\"65\", condition=\"unknown\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", job_name=\"helm-install-ipmi-exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546768950
Jul 17 03:33:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:54.424Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"msh-tsgx12\", asset_id=\"113\", condition=\"Available\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"kubernetes-dashboard\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"true\"}" timestamp=1689546768663
Jul 17 03:33:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:54.434Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_multicast_total\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth8fe690c2\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546768328
Jul 17 03:33:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:54.447Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx12\", asset_id=\"113\", cpu=\"54\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546763391
Jul 17 03:33:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:54.456Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_ports\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_endpoint=\"prometheus-kube-state-metrics\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", port_name=\"http\", port_number=\"8080\", port_protocol=\"TCP\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546764478
Jul 17 03:33:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:54.467Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"111\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546763391
Jul 17 03:33:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:54.471Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_errors_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"cadvisor\", interface=\"virtio_dign_c\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546769560
Jul 17 03:33:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:54.488Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_info\", asset=\"pcap-tsgx07\", asset_id=\"85\", configmap=\"prometheus-server\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546762058
Jul 17 03:33:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:54.492Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_spec_type\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"mrzcpd-exporter\", type=\"ClusterIP\", uid=\"65370c87-5d96-47da-ba63-3ff3de1a177d\"}" timestamp=1689546764478
Jul 17 03:33:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:54.545Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", container=\"coredns\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Async\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546762722
Jul 17 03:33:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:54.565Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_usage_bytes\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/run\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546763522
Jul 17 03:33:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:54.574Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"msh-tsgx12\", asset_id=\"113\", cpu=\"16\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546763391
Jul 17 03:33:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:54.575Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_reservation_limit_bytes\", asset=\"msh-tsgx12\", asset_id=\"113\", container=\"prometheus-pushgateway\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546760589
Jul 17 03:33:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:54.581Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"19\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546759409
Jul 17 03:33:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:54.619Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_usage_bytes\", asset=\"msh-tsgx12\", asset_id=\"113\", container=\"traefik\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546754975
Jul 17 03:33:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:54.636Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_multicast_packets_transmitted_total\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_10\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546763391
Jul 17 03:33:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:54.646Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2duuid-6551c5e8\\\\x2d9e70\\\\x2d499d\\\\x2dab9f\\\\x2d407e4afaf550.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689546760884
Jul 17 03:33:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:54.652Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_commands_rejected_calls_total\", asset=\"msh-tsgx12\", asset_id=\"113\", cmd=\"del\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546758659
Jul 17 03:33:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:54.661Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_commands_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cmd=\"zremrangebyscore\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546765184
Jul 17 03:33:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:54.708Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Pending\", pod=\"helm-install-mrzcpd-exporter-dnnph\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"cfb99589-b1d8-4b8b-b2c3-d91a1a70fd6b\"}" timestamp=1689546768663
Jul 17 03:33:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:54.724Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"lm_sensors.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"service\"}" timestamp=1689546771179
Jul 17 03:33:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:54.747Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:60-0000:60:03.1-0000:63:00.0-net-eno33np0.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689546760884
Jul 17 03:33:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:54.771Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"ftx_bits_total\", app=\"sapp4-0\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"nf_0_fw\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"mrzcpd-exporter\", serviceFunctionIndex=\"0\"}" timestamp=1689546771521
Jul 17 03:33:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:54.789Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"shm\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/b4e1ea9a1bdf46e1c045aa09601605abd5c1154bd72d8029126fbd9ca093e129/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546759409
Jul 17 03:33:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:54.809Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cpu=\"4\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546759409
Jul 17 03:33:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:54.823Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cpu=\"113\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546759409
Jul 17 03:33:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:54.836Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"plymouth-start.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"service\"}" timestamp=1689546760884
Jul 17 03:33:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:54.841Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_commands_total\", asset=\"msh-tsgx12\", asset_id=\"113\", cmd=\"info\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546758659
Jul 17 03:33:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:54.855Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_info\", asset=\"pcap-tsgx07\", asset_id=\"85\", cluster_ip=\"10.43.63.29\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"traefik\", uid=\"b613548a-61ed-46c2-8a8b-4a3078e25410\"}" timestamp=1689546762058
Jul 17 03:33:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:54.872Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"29\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546759409
Jul 17 03:33:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:54.879Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_ready_replicas\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"prometheus-pushgateway-bd476bcf9\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546762058
Jul 17 03:33:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:54.889Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"rpcbind.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689546760884
Jul 17 03:33:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:54.964Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db37\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546765184
Jul 17 03:33:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:55.039Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:00-0000:00:01.1-0000:01:00.1-infiniband_mad-umad1.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689546760884
Jul 17 03:33:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:55.058Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_annotations\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"prometheus-kube-state-metrics-6b768c7bf8\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546768950
Jul 17 03:33:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:55.100Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-ac7de96c1bdae86889e3ec214aaa0ccbf3c6c5d57fd25bea7f501cb63a497681.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"scope\"}" timestamp=1689546771179
Jul 17 03:33:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:55.113Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:e0-0000:e0:03.1-0000:e2:00.1-infiniband-mlx5_11.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689546771179
Jul 17 03:33:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:55.115Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_labels\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_endpoint=\"cm-policy-local-cache-redis-metrics\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546762058
Jul 17 03:33:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:55.133Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"metrics-server-54dc485875-jkvd8\", project=\"TSG-OS\", project_id=\"7\", reason=\"NodeAffinity\", service=\"prometheus-kube-state-metrics\", uid=\"0a0a8b78-3fe6-4b35-93e0-3e1bdd94459c\"}" timestamp=1689546762058
Jul 17 03:33:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:55.410Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_max_usage_bytes\", asset=\"pcap-tsgx07\", asset_id=\"85\", container=\"dashboard-metrics-scraper\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546761309
Jul 17 03:33:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:55.427Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_terminated_reason\", asset=\"pcap-tsgx07\", asset_id=\"85\", container=\"helm\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-tsg-traffic-engine-numa-0-clqgr\", project=\"TSG-OS\", project_id=\"7\", reason=\"Completed\", service=\"prometheus-kube-state-metrics\", uid=\"1ea39cdb-8ff7-4292-b608-b56268d85d12\"}" timestamp=1689546762058
Jul 17 03:33:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:55.447Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_info\", asset=\"pcap-tsgx07\", asset_id=\"85\", cluster_ip=\"10.43.92.93\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\", uid=\"a32fa5a3-7723-4779-847d-0a0dbd9b5546\"}" timestamp=1689546762058
Jul 17 03:33:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:55.611Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_terminated\", asset=\"pcap-tsgx07\", asset_id=\"85\", container=\"lb-tcp-443\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"svclb-traefik-b613548a-2ndxt\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"312ba98a-2da4-49c6-9feb-0611d5586d06\"}" timestamp=1689546762058
Jul 17 03:33:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:55.627Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cpu=\"12\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546759409
Jul 17 03:33:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:55.642Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"crond.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689546760884
Jul 17 03:33:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:56.607Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_rss\", asset=\"pcap-tsgx03\", asset_id=\"73\", container=\"prometheus-server-configmap-reload\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546768800
Jul 17 03:33:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:56.642Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_ports\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_endpoint=\"traefik\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", port_name=\"websecure\", port_number=\"8443\", port_protocol=\"TCP\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546773389
Jul 17 03:33:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:56.689Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_link_downed_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"mlx5_9\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546765883
Jul 17 03:33:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:56.692Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_created\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"prometheus-pushgateway\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546773389
Jul 17 03:33:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:56.732Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_drop_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"eno33np0\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546765883
Jul 17 03:33:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:57.026Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_name_assign_type\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth612726a5\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546765883
Jul 17 03:33:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:57.090Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_bytes_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens3f1np1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546765883
Jul 17 03:33:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:57.595Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_speed_bytes\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens3f1np1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546765883
Jul 17 03:33:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:57.676Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"40\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546765883
Jul 17 03:33:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:57.681Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_data_transmitted_bytes_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"mlx5_6\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546765883
Jul 17 03:33:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:57.696Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"21\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546765883
Jul 17 03:33:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:57.698Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_frame_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens1f1np1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546765883
Jul 17 03:33:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:57.709Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_packets_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens3f1np1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546765883
Jul 17 03:33:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:57.730Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"69\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546765883
Jul 17 03:33:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:57.836Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"nss-lookup.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"target\"}" timestamp=1689546763372
Jul 17 03:33:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:57.855Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-veth2b692087.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689546763372
Jul 17 03:33:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:57.928Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_owner\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", owner_is_controller=\"true\", owner_kind=\"ReplicaSet\", owner_name=\"coredns-f5d5fbf4d\", pod=\"coredns-f5d5fbf4d-d7dz4\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"06d9f977-06a7-4069-895f-e4ebce64d667\"}" timestamp=1689546773389
Jul 17 03:33:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:57.939Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db214\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546766475
Jul 17 03:33:58 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:58.032Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db237\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546766475
Jul 17 03:33:58 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:58.091Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"network-online.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"target\"}" timestamp=1689546763372
Jul 17 03:33:58 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:58.093Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_memory_used_overhead_bytes\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546766475
Jul 17 03:33:58 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:58.109Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_heap_inuse_bytes\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"ipmi-exporter\"}" timestamp=1689546774214
Jul 17 03:33:58 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:58.215Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_waiting\", asset=\"pcap-tsgx03\", asset_id=\"73\", container=\"metrics\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"cm-policy-local-cache-redis-master-69967b5879-sjdjw\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"5c522fdc-927c-4897-9ae6-09e4fb08b6fe\"}" timestamp=1689546773389
Jul 17 03:33:58 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:58.384Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_scrape_collector_success\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", collector=\"edac\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"OLAP-Node-Exporter-pcap-dellr740-dt001\", endpoint_id=\"251\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546777559
Jul 17 03:33:58 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:58.400Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_sockstat_TCP_orphan\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"OLAP-Node-Exporter-pcap-dellr740-dt001\", endpoint_id=\"251\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546777559
Jul 17 03:33:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:33:59.930Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_limit_bytes\", asset=\"msh-tsgx01\", asset_id=\"51\", container=\"prometheus-alertmanager-configmap-reload\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546774593
Jul 17 03:34:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:00.133Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_errors_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"cadvisor\", interface=\"bond0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546771566
Jul 17 03:34:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:00.178Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas_unavailable\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"ipmi-exporter\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546767425
Jul 17 03:34:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:00.203Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_bytes_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens4f0np0\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546769083
Jul 17 03:34:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:00.436Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"msh-tsgx01\", asset_id=\"51\", container=\"local-path-provisioner\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"iowaiting\"}" timestamp=1689546771926
Jul 17 03:34:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:00.447Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"43\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546769083
Jul 17 03:34:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:00.676Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_rss\", asset=\"msh-tsgx01\", asset_id=\"51\", container=\"dashboard-metrics-scraper\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546758230
Jul 17 03:34:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:00.780Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"msh-tsgx01\", asset_id=\"51\", container=\"lb-tcp-80\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"stopped\"}" timestamp=1689546759436
Jul 17 03:34:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:00.799Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_created\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-mrzcpd-exporter-6xctw\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"64e78458-5407-4009-8438-11206cb5f43d\"}" timestamp=1689546767425
Jul 17 03:34:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:00.844Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"37\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546769083
Jul 17 03:34:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:00.855Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_annotations\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"mrzcpd-exporter-5cf7f967cc\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546767425
Jul 17 03:34:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:00.861Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_processes\", asset=\"msh-tsgx01\", asset_id=\"51\", container=\"coredns\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546762609
Jul 17 03:34:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:01.058Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_info\", address=\"ee:37:7f:ce:2f:db\", asset=\"msh-tsgx01\", asset_id=\"51\", broadcast=\"ff:ff:ff:ff:ff:ff\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth8bc626f9\", duplex=\"full\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operstate=\"up\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546769083
Jul 17 03:34:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:01.118Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_bytes_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"cadvisor\", interface=\"ens2f1np1\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546771566
Jul 17 03:34:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:01.255Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_reservation_limit_bytes\", asset=\"msh-tsgx01\", asset_id=\"51\", container=\"traefik\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546774593
Jul 17 03:34:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:01.281Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas_updated\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"local-path-provisioner\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546767425
Jul 17 03:34:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:03.555Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-infiniband-umad2.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689546768447
Jul 17 03:34:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:03.610Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dracut-shutdown-onfailure.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689546768447
Jul 17 03:34:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:03.808Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_writes_total\", asset=\"msh-tsgx02\", asset_id=\"63\", container=\"dign-client\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546766645
Jul 17 03:34:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:03.808Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx02\", asset_id=\"63\", cpu=\"94\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546778318
Jul 17 03:34:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:03.838Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_receive_remote_physical_errors_total\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_7\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546778318
Jul 17 03:34:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:03.945Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"msh-tsgx02\", asset_id=\"63\", cpu=\"108\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546778318
Jul 17 03:34:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:03.994Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-88292df9a400697453f6b71cacb959f738a468b91fcfa47d9a0f641723a4f338-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689546768447
Jul 17 03:34:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:04.004Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda5\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", fstype=\"ext4\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/data\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546778318
Jul 17 03:34:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:04.029Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_tasks_max\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"import-state.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689546768447
Jul 17 03:34:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:04.087Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_errors_total\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", interface=\"eno34np1\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546770508
Jul 17 03:34:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:04.378Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"emergency.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"target\"}" timestamp=1689546768447
Jul 17 03:34:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:04.416Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"msh-tsgx02\", asset_id=\"63\", cpu=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546778318
Jul 17 03:34:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:04.444Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"msh-tsgx02\", asset_id=\"63\", cpu=\"105\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546778318
Jul 17 03:34:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:04.486Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_bytes_total\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"bond0\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546778318
Jul 17 03:34:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:04.752Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"msh-tsgx02\", asset_id=\"63\", condition=\"Available\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"tsg-traffic-engine-numa-0-firewall\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"false\"}" timestamp=1689546771412
Jul 17 03:34:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:04.781Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"shm\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/520ce657f334485cb6f710335a87b69c84743d6573b58274ee797277ab75c77b/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546778318
Jul 17 03:34:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:04.810Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_vl15_dropped_total\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_4\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546778318
Jul 17 03:34:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:04.828Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.grpc.v1.cri-sandboxes-5993fbf2779e312e5356bb7d0ce6ed2b47827433785c37242c5cee34c16aed28-shm.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689546768447
Jul 17 03:34:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:04.936Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"rpc-gssd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"service\"}" timestamp=1689546768447
Jul 17 03:34:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:04.959Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"swap.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"target\"}" timestamp=1689546768447
Jul 17 03:34:05 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:05.046Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"pcap-dellr740-dt002\", asset_id=\"33\", cpu=\"10\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"OLAP-Node-Exporter-pcap-dellr740-dt002\", endpoint_id=\"253\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546782497
Jul 17 03:34:05 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:05.205Z caller=dedupe.go:112 component=remote level=info remote_name=52db6f url=http://10.10.10.159:10090/nz-agent/prometheus/proxy/api/v1/write msg="Dropped sample for series that was not explicitly dropped via relabelling" ref=4436650
Jul 17 03:34:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:06.921Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_errs_total\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth5fb026f4\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546783910
Jul 17 03:34:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:07.086Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_cpu_seconds_total\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"NetworkManager.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689546773582
Jul 17 03:34:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:07.758Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"initrd-udevadm-cleanup-db.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"service\"}" timestamp=1689546773582
Jul 17 03:34:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:07.784Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"fcoe.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689546773582
Jul 17 03:34:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:07.940Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_packets_total\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"cadvisor\", interface=\"ens6f0np0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546782813
Jul 17 03:34:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:08.032Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db98\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546778290
Jul 17 03:34:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:08.068Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-c9cd2d343951639ada75f6442c40ab92c504e7e1a427ac84621b382c47742975.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"scope\"}" timestamp=1689546773582
Jul 17 03:34:09 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:09.908Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"jmx_config_reload_failure_total\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", module=\"DC-HBase\", module_id=\"3\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546788804
Jul 17 03:34:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:10.024Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_SplitTime_median\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Server\"}" timestamp=1689546788804
Jul 17 03:34:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:10.135Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_flushesQueuedCount\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"proxy_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"e82d9462faab70c1d607d81a60d135ec\", table=\"index_filename_proxy_hos_bucket\", type=\"index\"}" timestamp=1689546788804
Jul 17 03:34:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:10.590Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_storeFileSize\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"troubleshooting_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"9d47aadd56b020fb2f74604b5e408368\", table=\"index_time_troubleshooting_hos_bucket\", type=\"index\"}" timestamp=1689546788804
Jul 17 03:34:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:11.096Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_compactionsCompletedCount\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"assessment_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"91236b5787cc2a61a4ad1e613178dc07\", table=\"index_time_assessment_hos_bucket\", type=\"index\"}" timestamp=1689546788804
Jul 17 03:34:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:11.397Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_STATS_metric_deleteBatchTime_num_ops\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546790147
Jul 17 03:34:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:11.443Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_lastMajorCompactionAge\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", namespace=\"SYSTEM\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"71f969630421eb2700553877cf2f18d9\", table=\"LOG\", type=\"data\"}" timestamp=1689546790147
Jul 17 03:34:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:11.537Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_readRequestCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"firewall_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"963d87c0ad507a8707a36cf6f228f215\", table=\"index_partfile_firewall_hos_bucket\", type=\"index\"}" timestamp=1689546790147
Jul 17 03:34:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:11.635Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_scanCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"troubleshooting_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"606f43bb8ff18673fedf8b27c73d3dcd\", table=\"index_filename_troubleshooting_hos_bucket\", type=\"index\"}" timestamp=1689546790147
Jul 17 03:34:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:11.676Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_TotalCallTime_mean\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"IPC\"}" timestamp=1689546788804
Jul 17 03:34:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:11.689Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_replicaid\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"default\", project=\"OLAP\", project_id=\"3\", region=\"c3ec92d9078ffa0a16297912d37136cc\", table=\"index_filename_default\", type=\"index\"}" timestamp=1689546788804
Jul 17 03:34:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:11.743Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_194_4_metric_totalRequestCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546790147
Jul 17 03:34:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:11.832Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_PostIndexUpdateFailureTime_mean\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"PhoenixIndexer\"}" timestamp=1689546790147
Jul 17 03:34:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:11.857Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_STATS_metric_averageRegionSize\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546790147
Jul 17 03:34:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:11.870Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_STATS_metric_getTime_95th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546790147
Jul 17 03:34:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:11.886Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_193_1_metric_putBatchTime_max\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546790147
Jul 17 03:34:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:11.938Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_compactionsFailedCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"knowledge_base_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"8fb4b75f3ca29e745d6e59f60cf3b653\", table=\"index_partfile_knowledge_base_hos_bucket\", type=\"index\"}" timestamp=1689546790147
Jul 17 03:34:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:11.953Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Delete_99_9th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Server\"}" timestamp=1689546790147
Jul 17 03:34:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:12.011Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_scanCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"troubleshooting_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"054bf2784dec1caa24a332052fee8e3e\", table=\"index_filename_troubleshooting_hos_bucket\", type=\"index\"}" timestamp=1689546790147
Jul 17 03:34:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:12.195Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_193_1_metric_numReferenceFiles\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546790147
Jul 17 03:34:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:12.541Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_2_metric_deleteTime_25th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546790147
Jul 17 03:34:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:12.581Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_2_metric_compactionInputFileCount_25th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546790147
Jul 17 03:34:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:12.652Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_readRequestCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"firewall_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"874d1f49c3c0b3580409cad7d62139eb\", table=\"index_time_firewall_hos_bucket\", type=\"index\"}" timestamp=1689546790147
Jul 17 03:34:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:12.759Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_maxFlushQueueSize\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"session_record_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"1c69fefce8b241acd006936e0998b787\", table=\"index_time_session_record_hos_bucket\", type=\"index\"}" timestamp=1689546790147
Jul 17 03:34:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:12.834Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_compactionsQueuedCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"firewall_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"de3e9af7463fe090b6840184e6339540\", table=\"index_time_firewall_hos_bucket\", type=\"index\"}" timestamp=1689546790147
Jul 17 03:34:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:15.314Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx07\", asset_id=\"103\", container=\"firewall\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Discard\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689546773860
Jul 17 03:34:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:15.331Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_info\", asset=\"twa-tsgx07\", asset_id=\"103\", configmap=\"chart-content-systemd-exporter\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546788076
Jul 17 03:34:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:15.356Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_device_id\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens4f1np1\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546781230
Jul 17 03:34:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:15.374Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"1\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546781230
Jul 17 03:34:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:15.399Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-e6b2d9073791dc1d27c44ed22b234bdfae7ba7368e4c0e12b0d656712857f528-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689546787046
Jul 17 03:34:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:15.473Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"abrt-vmcore.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689546787046
Jul 17 03:34:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:15.495Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", state=\"uninterruptible\"}" timestamp=1689546785792
Jul 17 03:34:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:15.624Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_inodes_free\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda4\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546782789
Jul 17 03:34:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:15.639Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_name_assign_type\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethe34782c1\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546781230
Jul 17 03:34:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:15.661Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_address_assign_type\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens6f0np0\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546781230
Jul 17 03:34:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:15.717Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"twa-tsgx07\", asset_id=\"103\", container=\"firewall\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"cadvisor\", failure_type=\"pgfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"hierarchy\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689546773860
Jul 17 03:34:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:15.763Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"twa-tsgx07\", asset_id=\"103\", condition=\"Available\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"prometheus-server\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"unknown\"}" timestamp=1689546788076
Jul 17 03:34:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:15.825Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"46\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546781230
Jul 17 03:34:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:15.828Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"twa-tsgx07\", asset_id=\"103\", condition=\"false\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"mrzcpd-exporter-5cf7f967cc-95k2q\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"a324e348-d97e-48ff-833e-cccb478bc4df\"}" timestamp=1689546788076
Jul 17 03:34:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:15.941Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"mdmonitor.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"service\"}" timestamp=1689546787046
Jul 17 03:34:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:16.012Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_address_assign_type\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethc8675e90\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546781230
Jul 17 03:34:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:16.328Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"22\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546781230
Jul 17 03:34:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:16.391Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db95\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546786027
Jul 17 03:34:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:16.498Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx07\", asset_id=\"103\", container=\"dign-client\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Discard\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546771103
Jul 17 03:34:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:16.590Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"twa-tsgx07\", asset_id=\"103\", container=\"metrics\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"cadvisor\", failure_type=\"pgmajfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"hierarchy\"}" timestamp=1689546785003
Jul 17 03:34:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:16.643Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"126\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546781230
Jul 17 03:34:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:16.665Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_free_bytes\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"shm\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/fc00805fcb8c3c921b545edf164ebddf3a2d4157844624826f07ae710fca20fe/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546781230
Jul 17 03:34:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:16.713Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_info\", asset=\"twa-tsgx07\", asset_id=\"103\", container=\"helm\", container_id=\"containerd://6005c40307ba49e5e38d48f23e882db46e7559bcb487c0f4c1161ed4804c213b\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", image=\"docker.io/rancher/klipper-helm:v0.8.0-build20230510\", image_id=\"sha256:6f42df210d7faa0b6eb56f5f28bb061930c7834af1e7dd4b3586108ec6b1485b\", image_spec=\"rancher/klipper-helm:v0.8.0-build20230510\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-traefik-z7nkr\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"740bc957-d558-4d24-8190-9ceb22116933\"}" timestamp=1689546788076
Jul 17 03:34:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:16.728Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"48\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546781230
Jul 17 03:34:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:19.079Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"clickhouse_block_discard_bytes_sda\", asset=\"twa-dellr740-dt003\", asset_id=\"41\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"NC-Clickhouse-twa-dellr740-dt003\", endpoint_id=\"91\", module=\"NC-Clickhouse\", module_id=\"29\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546798300
Jul 17 03:34:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:19.452Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"8\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546797612
Jul 17 03:34:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:19.469Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"eno33np0\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546797612
Jul 17 03:34:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:19.489Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db228\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546791998
Jul 17 03:34:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:19.529Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_timer_last_trigger_seconds\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"mlocate-updatedb.timer\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689546784356
Jul 17 03:34:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:19.549Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"getty-pre.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"target\"}" timestamp=1689546784356
Jul 17 03:34:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:20.524Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_restarts_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", container=\"helm\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-cm-policy-local-cache-x5lq4\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"5859e672-b6bc-47e9-99af-51cf757d48a4\"}" timestamp=1689546793828
Jul 17 03:34:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:20.648Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_constraint_errors_transmitted_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"mlx5_1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546788263
Jul 17 03:34:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:20.656Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_bytes_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens4f1np1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546788263
Jul 17 03:34:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:20.672Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_daemonset_created\", asset=\"pcap-tsgx10\", asset_id=\"89\", daemonset=\"prometheus-node-exporter\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546793828
Jul 17 03:34:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:20.716Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_terminated\", asset=\"pcap-tsgx10\", asset_id=\"89\", container=\"helm\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-ipmi-exporter-ntwmm\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"6d124c79-91fd-435c-a730-222ebe3b4194\"}" timestamp=1689546793828
Jul 17 03:34:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:20.734Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_changes_total\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"eno2\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546797612
Jul 17 03:34:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:20.784Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_labels\", asset=\"twa-tsgx06\", asset_id=\"101\", configmap=\"prometheus-server\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546793949
Jul 17 03:34:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:20.823Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth86dfd1bd\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546788263
Jul 17 03:34:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:20.885Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", container=\"metrics-server\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"cadvisor\", failure_type=\"pgmajfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"container\"}" timestamp=1689546794967
Jul 17 03:34:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:20.912Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_sockets\", asset=\"pcap-tsgx10\", asset_id=\"89\", container=\"configmap-reloader\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546793829
Jul 17 03:34:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:20.959Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-89e2d284072c918459494a673de1758c0e35cfa8d573f12839747dd5eb98d0a1.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"scope\"}" timestamp=1689546790657
Jul 17 03:34:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:21.074Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_up\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546794840
Jul 17 03:34:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:21.397Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db117\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546794840
Jul 17 03:34:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:21.409Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_info\", asset=\"pcap-tsgx10\", asset_id=\"89\", created_by_kind=\"Job\", created_by_name=\"helm-install-prometheus\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", host_ip=\"10.10.10.170\", host_network=\"false\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", node=\"localhost\", nz_agent_id=\"3\", pod=\"helm-install-prometheus-fmhhw\", pod_ip=\"10.42.0.88\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"4f84a780-549d-4f40-bdec-aee8906d8427\"}" timestamp=1689546793828
Jul 17 03:34:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:21.433Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Pending\", pod=\"helm-install-traefik-b7tft\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"f8334eab-55b4-4187-9700-4ad2897e2d91\"}" timestamp=1689546793828
Jul 17 03:34:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:21.491Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_compressed_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens6f0np0\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546788263
Jul 17 03:34:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:21.496Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db23\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546794840
Jul 17 03:34:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:21.509Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_colls_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"vethd6f33409\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546788263
Jul 17 03:34:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:21.557Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2dpath-pci\\\\x2d0000:81:00.0\\\\x2dscsi\\\\x2d0:2:0:0.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689546790657
Jul 17 03:34:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:21.576Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-tmpfiles-clean.timer\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"timer\"}" timestamp=1689546790657
Jul 17 03:34:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:21.733Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dracut-initqueue.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689546790657
Jul 17 03:34:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:22.099Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-journal-flush.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"oneshot\", type=\"service\"}" timestamp=1689546790657
Jul 17 03:34:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:22.157Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"ipmi.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"service\"}" timestamp=1689546790657
Jul 17 03:34:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:22.173Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db62\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546794840
Jul 17 03:34:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:22.190Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"127\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546788263
Jul 17 03:34:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:22.358Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_gc_duration_seconds\", asset=\"msh-dellr740-olap004\", asset_id=\"17\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Spark-msh-dellr740-olap004\", endpoint_id=\"15\", module=\"NC-Spark\", module_id=\"47\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", quantile=\"0.5\"}" timestamp=1689546802253
Jul 17 03:34:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:23.283Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_frequency_max_hertz\", asset=\"msh-dellr740-olap004\", asset_id=\"17\", cpu=\"31\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Spark-msh-dellr740-olap004\", endpoint_id=\"15\", module=\"NC-Spark\", module_id=\"47\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546802253
Jul 17 03:34:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:23.516Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_labels\", asset=\"msh-tsgx09\", asset_id=\"55\", configmap=\"chart-content-cm-policy-local-cache\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546800068
Jul 17 03:34:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:23.636Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"nfs-server.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"service\"}" timestamp=1689546798416
Jul 17 03:34:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:23.642Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx09\", asset_id=\"55\", container=\"dign-client\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Write\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546794472
Jul 17 03:34:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:23.682Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_info\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_endpoint=\"cm-policy-local-cache-redis-master\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546800068
Jul 17 03:34:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:23.723Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Running\", pod=\"helm-install-configmap-reloader-84s59\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"47ba69e1-fd09-4e39-941a-89000685484e\"}" timestamp=1689546800068
Jul 17 03:34:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:23.743Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"polkit.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"service\"}" timestamp=1689546798416
Jul 17 03:34:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:23.996Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"msh-tsgx09\", asset_id=\"55\", cpu=\"66\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546794790
Jul 17 03:34:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:24.015Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db96\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546797019
Jul 17 03:34:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:24.139Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"msh-tsgx09\", asset_id=\"55\", cpu=\"9\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546794790
Jul 17 03:34:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:24.199Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx09\", asset_id=\"55\", cpu=\"118\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546794790
Jul 17 03:34:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:24.204Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_namespace_status_phase\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", phase=\"Terminating\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546791122
Jul 17 03:34:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:24.225Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"pcap-tsgx12\", asset_id=\"117\", condition=\"true\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"dign-client-lt9hj\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"a7373468-7e47-4929-8ff7-6d32b07661f2\"}" timestamp=1689546791122
Jul 17 03:34:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:24.228Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-2b50902878f015e6048315cd7747aafba261f013bc1962873baf8c64727ad42e.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"scope\"}" timestamp=1689546798416
Jul 17 03:34:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:24.271Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_working_set_bytes\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546802889
Jul 17 03:34:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:24.285Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx09\", asset_id=\"55\", cpu=\"70\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546794790
Jul 17 03:34:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:24.346Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_spec_strategy_rollingupdate_max_surge\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"metrics-server\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546791122
Jul 17 03:34:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:24.392Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_working_set_bytes\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"prometheus-server\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546797939
Jul 17 03:34:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:24.433Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"msh-tsgx09\", asset_id=\"55\", cpu=\"52\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546794790
Jul 17 03:34:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:24.453Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"pcap-tsgx12\", asset_id=\"117\", condition=\"unknown\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-tsg-traffic-engine-numa-0-mdqrn\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"86c61b36-b5a0-422f-be56-b68f56297496\"}" timestamp=1689546791122
Jul 17 03:34:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:24.471Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"36\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546800160
Jul 17 03:34:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:24.477Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_rate_bytes_per_second\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_10\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546794790
Jul 17 03:34:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:24.486Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_slave_repl_offset\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", master_host=\"192.168.20.157\", master_port=\"7112\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546797019
Jul 17 03:34:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:24.614Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db70\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546801795
Jul 17 03:34:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:24.863Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_daemonset_metadata_generation\", asset=\"pcap-tsgx12\", asset_id=\"117\", daemonset=\"kube-multus-ds\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546791122
Jul 17 03:34:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:25.019Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_system_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"lb-tcp-80\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546793056
Jul 17 03:34:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:25.036Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_info\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_endpoint=\"shaping-announce-port-tsg-traffic-engine-numa-0\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546791122
Jul 17 03:34:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:25.064Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_packets_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", interface=\"eno1\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546799275
Jul 17 03:34:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:25.158Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"abrt-ccpp.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"service\"}" timestamp=1689546791102
Jul 17 03:34:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:25.273Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_writes_bytes_total\", asset=\"msh-tsgx09\", asset_id=\"55\", container=\"traefik\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546795684
Jul 17 03:34:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:25.313Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_start_time_seconds\", asset=\"msh-tsgx09\", asset_id=\"55\", container=\"metrics\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546798586
Jul 17 03:34:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:25.489Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db216\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546801795
Jul 17 03:34:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:25.503Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"20\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546800160
Jul 17 03:34:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:25.521Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-virtual-net-vethfc5d9f9d.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689546791102
Jul 17 03:34:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:25.539Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_size_bytes\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"tmpfs\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546800160
Jul 17 03:34:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:25.666Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_carrier_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens7f1np1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546800160
Jul 17 03:34:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:25.682Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"57\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546800160
Jul 17 03:34:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:26.023Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_metadata_resource_version\", asset=\"twa-tsgx09\", asset_id=\"107\", configmap=\"chart-content-cm-policy-local-cache\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546798744
Jul 17 03:34:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:26.392Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"28\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546795810
Jul 17 03:34:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:26.467Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_load_average_10s\", asset=\"twa-tsgx09\", asset_id=\"107\", container=\"dign-client\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546799999
Jul 17 03:34:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:26.488Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-veth1e7f8cb3.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689546793907
Jul 17 03:34:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:26.509Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_tasks_max\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"abrt-xorg.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689546793907
Jul 17 03:34:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:26.516Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"126\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546795810
Jul 17 03:34:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:26.519Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"125\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546795810
Jul 17 03:34:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:26.602Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_up\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens1f0np0\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546795810
Jul 17 03:34:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:26.631Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2did-wwn\\\\x2d0x62cea7f08346b9002c25f712ca485ab5\\\\x2dpart1.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689546793907
Jul 17 03:34:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:26.700Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_info\", address=\"ea:84:a2:ab:6b:b9\", asset=\"twa-tsgx09\", asset_id=\"107\", broadcast=\"ff:ff:ff:ff:ff:ff\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth0fc63319\", duplex=\"full\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operstate=\"up\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546795810
Jul 17 03:34:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:26.728Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"twa-tsgx09\", asset_id=\"107\", container=\"dign-client\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"running\"}" timestamp=1689546799999
Jul 17 03:34:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:26.752Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link_mode\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens1f0np0\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546795810
Jul 17 03:34:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:26.773Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_bytes_total\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens4f3np3\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546795810
Jul 17 03:34:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:27.599Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_owner\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", owner_is_controller=\"true\", owner_kind=\"ReplicaSet\", owner_name=\"metrics-server-54dc485875\", pod=\"metrics-server-54dc485875-mqqt7\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"b0c47859-1fba-4157-b772-f4e3a8229921\"}" timestamp=1689546795682
Jul 17 03:34:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:27.622Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"atd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"service\"}" timestamp=1689546794114
Jul 17 03:34:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:27.853Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_cpu_shares\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"dign-server-dns\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546804841
Jul 17 03:34:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:27.895Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_info\", asset=\"twa-tsgx09\", asset_id=\"107\", created_by_kind=\"ReplicaSet\", created_by_name=\"kubernetes-dashboard-66dd5b456\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", host_ip=\"10.10.30.169\", host_network=\"false\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", node=\"localhost\", nz_agent_id=\"3\", pod=\"kubernetes-dashboard-66dd5b456-bgg44\", pod_ip=\"10.42.0.72\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"c8850d1a-9bcd-4bf0-adad-91f75a60d9cc\"}" timestamp=1689546798744
Jul 17 03:34:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:27.970Z caller=dedupe.go:112 component=remote level=error remote_name=52db6f url=http://10.10.10.159:10090/nz-agent/prometheus/proxy/api/v1/write msg="non-recoverable error" count=1000 exemplarCount=0 err="server returned HTTP status 400 : out of order sample"
Jul 17 03:34:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:28.043Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"117\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546795810
Jul 17 03:34:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:28.066Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"110\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546795810
Jul 17 03:34:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:28.111Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_usage_seconds_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"total\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689546803675
Jul 17 03:34:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:28.124Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_writes_total\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"prometheus-pushgateway\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546796973
Jul 17 03:34:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:28.140Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"82\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546802882
Jul 17 03:34:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:28.163Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-coredump@0.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"simple\", type=\"service\"}" timestamp=1689546794114
Jul 17 03:34:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:28.185Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"120\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546802882
Jul 17 03:34:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:28.256Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_reads_total\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"lb-tcp-80\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546796143
Jul 17 03:34:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:28.278Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_state_started\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"helm\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-delete-tsg-traffic-engine-numa-1-bhwwl\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"b8eea581-ef7b-4e4f-a2ad-1d45302a2df4\"}" timestamp=1689546795682
Jul 17 03:34:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:28.303Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"java_lang_MemoryPool_PeakUsage_committed\", asset=\"pcap-dellr740-dt002\", asset_id=\"33\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt002-Hmaster\", endpoint_id=\"345\", module=\"DC-HBase\", module_id=\"3\", name=\"Par Survivor Space\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546807137
Jul 17 03:34:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:28.317Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_restarts_total\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"prometheus-server-configmap-reload\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-server-6ff784d758-bvjp8\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"0bdb24ec-e0f8-44e3-8d3f-a437820bc4e9\"}" timestamp=1689546795682
Jul 17 03:34:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:28.414Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_free_bytes\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"shm\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/e9354d05869568b84eed6d0c46c96d263808d521e197a03555bb7e281789e4a5/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546802882
Jul 17 03:34:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:28.432Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_packets_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens6f1np1\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546802882
Jul 17 03:34:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:28.537Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_rate_bytes_per_second\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_4\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546802882
Jul 17 03:34:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:28.551Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_mtu_bytes\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens4f3np3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546802882
Jul 17 03:34:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:28.642Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_threads\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"configmap-reloader\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546801552
Jul 17 03:34:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:28.675Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_unicast_packets_received_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_10\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546802882
Jul 17 03:34:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:28.691Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-rdma-devices-mlx5_7:1-umad.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689546794114
Jul 17 03:34:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:28.713Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"106\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546802882
Jul 17 03:34:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:28.729Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"6\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546802882
Jul 17 03:34:29 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:29.005Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_frame_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth3ea9a2cc\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546802882
Jul 17 03:34:29 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:29.110Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Succeeded\", pod=\"local-path-provisioner-957fdf8bc-hswhg\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"99c87350-195b-4ca1-93bd-9a804bf4d300\"}" timestamp=1689546795682
Jul 17 03:34:29 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:29.123Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_files_free\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"shm\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/f93f6d15ed0e3fc2be45c47643a2fcaf32cd0cc71af5881df789c31d481b6387/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546802882
Jul 17 03:34:29 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:29.152Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"43\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546802882
Jul 17 03:34:30 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:30.664Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"jmx_config_reload_failure_total\", asset=\"twa-dellr740-dt002\", asset_id=\"39\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt002-Hmaster\", endpoint_id=\"349\", module=\"DC-HBase\", module_id=\"3\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546809701
Jul 17 03:34:30 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:30.707Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_numMasterWALs\", asset=\"twa-dellr740-dt002\", asset_id=\"39\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt002-Hmaster\", endpoint_id=\"349\", module=\"DC-HBase\", module_id=\"3\", name=\"Master\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Procedure\"}" timestamp=1689546809701
Jul 17 03:34:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:31.360Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db164\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546810141
Jul 17 03:34:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:31.385Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-infiniband-umad6.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689546801015
Jul 17 03:34:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:31.554Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_frame_total\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"br_dign_c\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546800939
Jul 17 03:34:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:31.570Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"ext4\", name=\"var-lib-kubelet-pods-d8012c00\\\\x2daaec\\\\x2d4f56\\\\x2da35b\\\\x2dad6f4d1ef590-volume\\\\x2dsubpaths-config\\\\x2dvolume-firewall-1.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689546801015
Jul 17 03:34:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:31.601Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-rdma-devices-mlx5_6:1-umad.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689546806907
Jul 17 03:34:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:31.727Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:c0-0000:c0:01.1-0000:c1:00.0-infiniband_mad-umad8.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689546806907
Jul 17 03:34:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:31.754Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-sda1.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689546801015
Jul 17 03:34:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:31.775Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_cpu_shares\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"dign-server-web\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546802009
Jul 17 03:34:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:31.786Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sshd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"service\"}" timestamp=1689546801015
Jul 17 03:34:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:31.793Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_created\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", job_name=\"helm-delete-tsg-traffic-engine-numa-1\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546799276
Jul 17 03:34:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:31.813Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"44\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546800791
Jul 17 03:34:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:31.831Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_bytes_total\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", interface=\"ens4f3np3\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546797067
Jul 17 03:34:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:31.833Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"72\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546800791
Jul 17 03:34:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:31.850Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_terminated\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"firewall\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", pod=\"tsg-traffic-engine-numa-0-firewall-99cddfc69-25wpt\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"abf27d03-a067-448d-81a8-11df769a2208\"}" timestamp=1689546799276
Jul 17 03:34:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:31.859Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"70\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546800791
Jul 17 03:34:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:31.867Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"tmpfs\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/data/var/lib/kubelet/pods/1ae789f3-8760-4836-9727-170790799776/volumes/kubernetes.io~projected/kube-api-access-zpppq\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546800791
Jul 17 03:34:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:31.883Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"58\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546799093
Jul 17 03:34:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:31.923Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db186\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546796711
Jul 17 03:34:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:31.977Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"24\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546800791
Jul 17 03:34:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:31.994Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db241\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546796711
Jul 17 03:34:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:32.001Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_io_time_seconds_total\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda5\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546797067
Jul 17 03:34:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:32.018Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_mapped_file\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689546797764
Jul 17 03:34:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:32.022Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"mrzcpd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689546802644
Jul 17 03:34:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:32.038Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.grpc.v1.cri-sandboxes-a98f3e8875a4e0eb7a00a1677df4b100c59d90548cb5d22905911c3bc6896c33-shm.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689546802644
Jul 17 03:34:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:32.074Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-br_dign_c.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689546802644
Jul 17 03:34:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:32.154Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"iscsi-onboot.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"service\"}" timestamp=1689546802644
Jul 17 03:34:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:32.171Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filefd_allocated\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546800791
Jul 17 03:34:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:32.203Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_metadata_generation\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"cm-policy-local-cache-redis-master\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546810436
Jul 17 03:34:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:32.247Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_info\", asset=\"msh-tsgx05\", asset_id=\"67\", configmap=\"chart-content-configmap-reloader\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546804068
Jul 17 03:34:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:32.267Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_annotations\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"chart-values-ipmi-exporter\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546810436
Jul 17 03:34:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:32.284Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"twa-tsgx03\", asset_id=\"95\", condition=\"Progressing\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"kubernetes-dashboard\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"false\"}" timestamp=1689546810436
Jul 17 03:34:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:32.303Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-rfkill.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689546806907
Jul 17 03:34:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:32.311Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_dormant\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth1618597b\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546799093
Jul 17 03:34:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:32.324Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-sda4.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689546806907
Jul 17 03:34:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:32.332Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_compressed_total\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth1ccea153\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546800791
Jul 17 03:34:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:32.349Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"119\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546799093
Jul 17 03:34:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:32.409Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_total_error_replies\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546806438
Jul 17 03:34:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:32.453Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"117\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546800791
Jul 17 03:34:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:32.484Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"11\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546800939
Jul 17 03:34:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:32.492Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_down_changes_total\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth430fd9d5\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546800791
Jul 17 03:34:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:32.507Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_queue_length\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens2f0np0\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546800791
Jul 17 03:34:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:32.512Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_net_dev_group\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens7f0np0\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546800791
Jul 17 03:34:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:32.526Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:40-0000:40:01.1-0000:41:00.0-infiniband-mlx5_4.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689546802644
Jul 17 03:34:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:32.530Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_errs_total\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth1d757172\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546800939
Jul 17 03:34:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:32.536Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_file_descriptors\", asset=\"twa-tsgx03\", asset_id=\"95\", container=\"telegraf\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689546793126
Jul 17 03:34:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:32.555Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_mapped_file\", asset=\"twa-tsgx03\", asset_id=\"95\", container=\"prometheus-server-configmap-reload\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546795428
Jul 17 03:34:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:32.617Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-virtual-net-vethb4349618.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689546806907
Jul 17 03:34:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:32.805Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_annotations\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_endpoint=\"systemd-exporter\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546799276
Jul 17 03:34:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:32.823Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"ipmi.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"service\"}" timestamp=1689546802644
Jul 17 03:34:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:32.832Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_rdb_last_bgsave_status\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546809260
Jul 17 03:34:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:32.863Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-a217e936a61c18e0031a317ea83f896be4d78656ada6cb083399cfc92ddce3e9.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"scope\"}" timestamp=1689546807691
Jul 17 03:34:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:32.867Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx03\", asset_id=\"95\", container=\"coredns\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Sync\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546803039
Jul 17 03:34:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:32.870Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"overlay\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-7398125dcac036e9467a5e64699d8816e5ffa907fc39919e5cadea0af235e691-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689546807691
Jul 17 03:34:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:32.884Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cryptsetup.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"target\"}" timestamp=1689546807691
Jul 17 03:34:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:32.901Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_memory_Inactive_anon_bytes\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546800605
Jul 17 03:34:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:32.928Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_compressed_total\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"virtio_dign_s\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546799093
Jul 17 03:34:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:32.964Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"125\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546799093
Jul 17 03:34:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:33.009Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"plymouth-start.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"service\"}" timestamp=1689546806907
Jul 17 03:34:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:33.026Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-rdma-devices-mlx5_8:1-umad.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689546806907
Jul 17 03:34:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:33.109Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_info\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"prometheus-node-exporter\", container_id=\"containerd://57266186cfd03f584cdff162a2dc75d868b8047b4b3ea1974dee174eec0c64c6\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", image=\"quay.io/prometheus/node-exporter:v1.3.1\", image_id=\"sha256:1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26\", image_spec=\"quay.io/prometheus/node-exporter:v1.3.1\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-node-exporter-kpcmt\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"d4700480-e588-4509-b70f-b20c4e312a51\"}" timestamp=1689546799276
Jul 17 03:34:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:33.120Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db57\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546809260
Jul 17 03:34:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:33.135Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_keyspace_misses_total\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546809260
Jul 17 03:34:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:33.170Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_id\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth792cff4d\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546800605
Jul 17 03:34:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:33.203Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-a24d3320b0f9c1b095afb4f8fcb4d3f4ab9f079a892d5879c89d4d1dc21e51ba-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689546802644
Jul 17 03:34:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:33.216Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_info\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"chart-values-prometheus\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546799276
Jul 17 03:34:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:33.232Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:60-0000:60:03.1-0000:63:00.3-net-eno36np3.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689546802644
Jul 17 03:34:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:33.255Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2did-scsi\\\\x2d362cea7f0add57d002c2c229b337423e6\\\\x2dpart2.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689546802644
Jul 17 03:34:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:33.282Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_persistentvolumeclaim_info\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", persistentvolumeclaim=\"prometheus-server\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", storageclass=\"local-path\", volumename=\"pvc-85ed777f-1d1b-4f74-a3c5-b6361a7e91b9\"}" timestamp=1689546799276
Jul 17 03:34:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:33.329Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"99\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546800791
Jul 17 03:34:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:33.376Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.grpc.v1.cri-sandboxes-e41815df358d5e53d4c4527c647f58609f01f0d5beaa493c16a5e9f97a73fc72-shm.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689546802644
Jul 17 03:34:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:33.447Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"twa-tsgx05\", asset_id=\"99\", container=\"firewall\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", state=\"stopped\"}" timestamp=1689546795889
Jul 17 03:34:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:33.491Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_transmit_wait_total\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_9\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546800605
Jul 17 03:34:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:33.522Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"k3s.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"notify\", type=\"service\"}" timestamp=1689546807691
Jul 17 03:34:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:33.748Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_fifo_total\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"lo\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546800605
Jul 17 03:34:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:33.777Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db34\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546809260
Jul 17 03:34:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:34.217Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_terminated_reason\", asset=\"twa-tsgx05\", asset_id=\"99\", container=\"helm\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-traefik-crd-bf9k4\", project=\"TSG-OS\", project_id=\"7\", reason=\"Completed\", service=\"prometheus-kube-state-metrics\", uid=\"39ad39af-b9c7-4eb1-a7db-349fcd19b742\"}" timestamp=1689546799894
Jul 17 03:34:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:34.293Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"16\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546800605
Jul 17 03:34:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:34.372Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_start_time\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"dign-client-p9lfb\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"faaa93b7-eae4-4c14-96b8-0ca5e8ca7cad\"}" timestamp=1689546799894
Jul 17 03:34:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:34.403Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_link_error_recovery_total\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_9\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546800605
Jul 17 03:34:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:40.539Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_symbol_error_total\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_1\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546811647
Jul 17 03:34:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:40.550Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"75\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546811647
Jul 17 03:34:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:40.698Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db238\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546813827
Jul 17 03:34:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:40.844Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_packets_total\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"eno35np2\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546811647
Jul 17 03:34:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:40.872Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_queue_length\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens8f0np0\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546811647
Jul 17 03:34:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:40.894Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"90\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546811647
Jul 17 03:34:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:41.418Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dracut-shutdown-onfailure.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"service\"}" timestamp=1689546815764
Jul 17 03:34:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:41.438Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-netns-cni\\\\x2d8fd961b5\\\\x2db53c\\\\x2d8469\\\\x2d5bae\\\\x2d0fdcf51ed830.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689546815764
Jul 17 03:34:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:41.551Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-6ad56cef67554bb9f0441c387ad1d7490cd1e789363bc85b4fdae15b90b2bbdd-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689546815764
Jul 17 03:34:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:41.584Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_persistentvolume_status_phase\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", persistentvolume=\"pvc-49234523-0cb4-4f93-a353-6b9f8a2112c6\", phase=\"Released\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546820034
Jul 17 03:34:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:41.603Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_info\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"chart-values-traefik-crd\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546820034
Jul 17 03:34:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:41.764Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"1\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546811647
Jul 17 03:34:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:41.780Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_compressed_total\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth3c119b31\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546811647
Jul 17 03:34:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:41.796Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_errs_total\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"eno34np1\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546811647
Jul 17 03:34:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:41.831Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_compactionsCompletedCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"default\", project=\"OLAP\", project_id=\"3\", region=\"3e6cb7a570cd3be87bb804a41b3d1d77\", table=\"index_partfile_default\", type=\"index\"}" timestamp=1689546819757
Jul 17 03:34:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:41.853Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_created\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"ipmi-exporter\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546820034
Jul 17 03:34:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:41.888Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_numBytesCompactedCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"firewall_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"bcee559236e0be70ad0a1e9d2a915a48\", table=\"index_partfile_firewall_hos_bucket\", type=\"index\"}" timestamp=1689546819757
Jul 17 03:34:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:41.888Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-tsg-traffic-engine-numa-1-m7z2d\", project=\"TSG-OS\", project_id=\"7\", reason=\"Evicted\", service=\"prometheus-kube-state-metrics\", uid=\"3e240bd7-471b-4018-83da-72f34cec6151\"}" timestamp=1689546820034
Jul 17 03:34:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:41.970Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_protocol_type\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth607a2032\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546811647
Jul 17 03:34:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:42.007Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_TASK_metric_incrementTime_75th_percentile\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546819757
Jul 17 03:34:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:42.193Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_writeRequestCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"proxy_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"647386ee248901a50fcc87544d60e617\", table=\"index_time_proxy_hos_bucket\", type=\"index\"}" timestamp=1689546819757
Jul 17 03:34:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:42.959Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_name_assign_type\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"eno36np3\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546816242
Jul 17 03:34:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:42.989Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_fully_labeled_replicas\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"dashboard-metrics-scraper-5b94897dc9\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546815154
Jul 17 03:34:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:42.992Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"unbound-anchor.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"service\"}" timestamp=1689546820671
Jul 17 03:34:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:43.413Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_load_average_10s\", asset=\"pcap-tsgx01\", asset_id=\"71\", container=\"dign-client\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546820596
Jul 17 03:34:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:43.514Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546816242
Jul 17 03:34:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:43.853Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"network.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"target\"}" timestamp=1689546820671
Jul 17 03:34:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:43.966Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_system_seconds_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", container=\"metrics-server\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546809047
Jul 17 03:34:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:43.981Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"ipmi-exporter\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546815154
Jul 17 03:34:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:43.996Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_labels\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", job_name=\"helm-install-systemd-exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546815154
Jul 17 03:34:46 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:46.038Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"java_lang_Memory_HeapMemoryUsage_init\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-Yarn-pcap-dellr740-dt001-ResourceManager\", endpoint_id=\"135\", module=\"DC-Yarn\", module_id=\"1\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546825217
Jul 17 03:34:48 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:48.630Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_1_metric_scanSize_90th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546827438
Jul 17 03:34:48 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:48.741Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_putCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"assessment_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"43c96dd21aa1238d6db7226daea0110e\", table=\"index_filename_assessment_hos_bucket\", type=\"index\"}" timestamp=1689546827438
Jul 17 03:34:48 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:48.764Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_PostIndexUpdateFailureTime_95th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"PhoenixIndexer\"}" timestamp=1689546827438
Jul 17 03:34:48 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:48.781Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_default_metric_compactedOutputBytes\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546827438
Jul 17 03:34:48 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:48.936Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_readRequestCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"troubleshooting_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"4e4a10d75a0bbcfbfd0bc2ef1602f3bf\", table=\"index_time_troubleshooting_hos_bucket\", type=\"index\"}" timestamp=1689546827438
Jul 17 03:34:48 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:48.954Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_flushesQueuedCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"troubleshooting_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"8f3fd78be39bfba7066de032e9a0dd42\", table=\"index_partfile_troubleshooting_hos_bucket\", type=\"index\"}" timestamp=1689546827438
Jul 17 03:34:48 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:48.990Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_4_metric_flushMemstoreSize_98th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546827438
Jul 17 03:34:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:49.008Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_numReferenceFiles\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"proxy_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"12c8bac854969dda8707b0a6c52163c3\", table=\"index_time_proxy_hos_bucket\", type=\"index\"}" timestamp=1689546827438
Jul 17 03:34:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:49.014Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_194_1_metric_deleteTime_25th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546827438
Jul 17 03:34:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:49.028Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_193_1_metric_majorCompactionOutputFileCount_max\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546827438
Jul 17 03:34:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:49.032Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_tsg_galaxy_table_relation_user_teid_metric_deleteBatchTime_75th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546827438
Jul 17 03:34:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:49.036Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_minStoreFileAge\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"knowledge_base_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"e122c8dbd33832397f71e41a35867a3f\", table=\"index_partfile_knowledge_base_hos_bucket\", type=\"index\"}" timestamp=1689546827438
Jul 17 03:34:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:49.040Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_numFilesCompactedCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"default\", project=\"OLAP\", project_id=\"3\", region=\"1dc35781b98addf9940b9004125b0307\", table=\"index_filename_default\", type=\"index\"}" timestamp=1689546827438
Jul 17 03:34:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:49.044Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_PreIndexUpdateFailureTime_max\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"PhoenixIndexer\"}" timestamp=1689546827438
Jul 17 03:34:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:49.061Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_3_metric_putTime_mean\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546827438
Jul 17 03:34:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:49.078Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_filteredReadRequestCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"firewall_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"e4331eab37a02574cfac02404cf8f4bf\", table=\"index_filename_firewall_hos_bucket\", type=\"index\"}" timestamp=1689546827438
Jul 17 03:34:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:49.096Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_avgStoreFileAge\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"proxy_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"5648ed44ac1d1626415660c3c36fbd73\", table=\"index_time_proxy_hos_bucket\", type=\"index\"}" timestamp=1689546827438
Jul 17 03:34:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:49.102Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_4_metric_getTime_min\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546827438
Jul 17 03:34:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:49.111Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_compactionsFailedCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"proxy_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"f0123c96ecf2d946605d86423433d8fc\", table=\"index_partfile_proxy_hos_bucket\", type=\"index\"}" timestamp=1689546827438
Jul 17 03:34:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:49.118Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_compactionsCompletedCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"29d0f7474cbf29ea23802548355e9be7\", table=\"session_record_hos_bucket\", type=\"data\"}" timestamp=1689546827438
Jul 17 03:34:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:49.192Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_incrementCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"assessment_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"fc609c80cb371f2d006154dd6cbfcc14\", table=\"index_partfile_assessment_hos_bucket\", type=\"index\"}" timestamp=1689546827438
Jul 17 03:34:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:49.507Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_getCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"rtp_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"f8837ba4ff20870416f2ed1fe9d1f4d0\", table=\"index_filename_rtp_hos_bucket\", type=\"index\"}" timestamp=1689546827438
Jul 17 03:34:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:49.614Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_tsg_galaxy_table_relation_account_framedip_metric_scanSize_75th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546827438
Jul 17 03:34:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:49.891Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_194_2_metric_majorCompactionOutputSize_num_ops\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546827438
Jul 17 03:34:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:50.064Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", cpu=\"42\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-olap003\", endpoint_id=\"235\", mode=\"nice\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"yarn_resourcemanager,zookeeper,nacos,kafka,spark,druid_data,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546829713
Jul 17 03:34:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:50.090Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_user_seconds_total\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"local-path-provisioner\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546815120
Jul 17 03:34:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:50.140Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_working_set_bytes\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"redis\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546805154
Jul 17 03:34:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:50.168Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_errors_total\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", interface=\"eth0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546818824
Jul 17 03:34:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:50.209Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_terminated\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"telegraf\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", pod=\"tsg-traffic-engine-numa-0-firewall-7886f9896b-4qb7z\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"8bb79bea-895f-4fbf-bba7-ff346d91b1a0\"}" timestamp=1689546819668
Jul 17 03:34:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:50.576Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_annotations\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"dashboard-metrics-scraper\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546821568
Jul 17 03:34:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:50.633Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_gc_duration_seconds\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-olap003\", endpoint_id=\"235\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"yarn_resourcemanager,zookeeper,nacos,kafka,spark,druid_data,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\", quantile=\"0.75\"}" timestamp=1689546829713
Jul 17 03:34:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:50.649Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"26\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546826709
Jul 17 03:34:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:50.672Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"98\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546826709
Jul 17 03:34:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:50.689Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-6e4f2f4687bae3bbca1a2a2cbbc5a63313dfd06a46f8ecb30e6b1173c86a1f59-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689546826554
Jul 17 03:34:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:50.699Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_reads_merged_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/6e4f2f4687bae3bbca1a2a2cbbc5a63313dfd06a46f8ecb30e6b1173c86a1f59/shm\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546823919
Jul 17 03:34:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:50.711Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"57\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546826709
Jul 17 03:34:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:50.729Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_cpu_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"rpcbind.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689546826554
Jul 17 03:34:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:50.764Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dracut-pre-mount.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689546826554
Jul 17 03:34:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:50.774Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_reads_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/var/lib/kubelet/pods/bc493ff5-41bc-461a-a60b-e1fa21aeebe7/volumes/kubernetes.io~projected/kube-api-access-nnstn\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546823919
Jul 17 03:34:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:50.860Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_write_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"overlay_0-307\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546823919
Jul 17 03:34:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:50.866Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"81\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546826709
Jul 17 03:34:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:50.902Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"32\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546821797
Jul 17 03:34:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:50.937Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", fstype=\"ext4\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/data/var/lib/kubelet/pods/8bb79bea-895f-4fbf-bba7-ff346d91b1a0/volume-subpaths/telegraf/telegraf/0\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546821797
Jul 17 03:34:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:50.938Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_fifo_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens2f1np1\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546826709
Jul 17 03:34:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:50.956Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_hwmon_chip_names\", asset=\"twa-tsgx10\", asset_id=\"109\", chip=\"pci0000:00_0000:00:19_3\", chip_name=\"k10temp\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546821797
Jul 17 03:34:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:50.969Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas_updated\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"mrzcpd-exporter\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546821568
Jul 17 03:34:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:51.024Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_terminated_reason\", asset=\"twa-tsgx08\", asset_id=\"105\", container=\"helm\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-ipmi-exporter-pspfb\", project=\"TSG-OS\", project_id=\"7\", reason=\"Completed\", service=\"prometheus-kube-state-metrics\", uid=\"8d60fd58-fc4a-4fec-a27c-e595a24480b5\"}" timestamp=1689546821568
Jul 17 03:34:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:51.044Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_init_container_resource_requests\", asset=\"twa-tsgx08\", asset_id=\"105\", container=\"install-multus-binary\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", node=\"localhost\", nz_agent_id=\"3\", pod=\"kube-multus-ds-rvwzq\", project=\"TSG-OS\", project_id=\"7\", resource=\"memory\", service=\"prometheus-kube-state-metrics\", uid=\"132d373d-8755-45d4-97c7-04927f0acd43\", unit=\"byte\"}" timestamp=1689546821568
Jul 17 03:34:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:51.059Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"overlay\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-bae299751af06ba2db63d4d41dde020be363f2548c76ab4582c68e583e80aa09-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689546821057
Jul 17 03:34:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:51.120Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"35\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546826709
Jul 17 03:34:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:51.135Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_info\", address=\"3a:93:f3:cf:c6:0c\", asset=\"twa-tsgx08\", asset_id=\"105\", broadcast=\"ff:ff:ff:ff:ff:ff\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"br_dign_c\", duplex=\"unknown\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operstate=\"up\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546826709
Jul 17 03:34:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:51.194Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_running\", asset=\"twa-tsgx08\", asset_id=\"105\", container=\"helm\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-mrzcpd-exporter-zjhth\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"a88ca435-a295-43d3-bffd-a3842ce86c5e\"}" timestamp=1689546821568
Jul 17 03:34:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:51.224Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_sector_writes_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/var/lib/kubelet/pods/32707ee1-6f45-4d2c-8e0e-78edc233adc3/volumes/kubernetes.io~projected/kube-api-access-x64jh\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546823919
Jul 17 03:34:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:51.226Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_writes_total\", asset=\"twa-tsgx08\", asset_id=\"105\", container=\"prometheus-server\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546822724
Jul 17 03:34:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:51.241Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"twa-tsgx08\", asset_id=\"105\", condition=\"unknown\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"cm-policy-local-cache-redis-master-69967b5879-g5dr8\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"ee4ec55a-f496-4629-b299-cf918d7658cb\"}" timestamp=1689546821568
Jul 17 03:34:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:51.282Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_core_throttles_total\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", core=\"3\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-olap003\", endpoint_id=\"235\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"yarn_resourcemanager,zookeeper,nacos,kafka,spark,druid_data,yarn_nodemanager\", package=\"1\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546829713
Jul 17 03:34:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:51.286Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_tasks_max\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"fcoe.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689546821057
Jul 17 03:34:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:51.288Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", cpu=\"10\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-olap003\", endpoint_id=\"235\", mode=\"steal\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"yarn_resourcemanager,zookeeper,nacos,kafka,spark,druid_data,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546829713
Jul 17 03:34:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:51.381Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_free_bytes\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"shm\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/57586ef2b12d3be3dde6eb77d147247c3bda45d0866adf6b2d51c2372d8963e5/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546826709
Jul 17 03:34:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:51.461Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_cpu_period\", asset=\"twa-tsgx08\", asset_id=\"105\", container=\"local-path-provisioner\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546827561
Jul 17 03:34:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:51.544Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_commands_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cmd=\"unlink\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546827556
Jul 17 03:34:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:51.596Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db126\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546827556
Jul 17 03:34:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:53.360Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx08\", asset_id=\"65\", cpu=\"108\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546828328
Jul 17 03:34:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:53.382Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx08\", asset_id=\"65\", cpu=\"64\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546828328
Jul 17 03:34:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:53.444Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-vetha6b2596c.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689546832135
Jul 17 03:34:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:53.461Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_rss\", asset=\"msh-tsgx12\", asset_id=\"113\", container=\"kube-multus\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546814876
Jul 17 03:34:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:53.517Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth3449d361\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546828328
Jul 17 03:34:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:53.545Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Running\", pod=\"traefik-64f55bb67d-mchmv\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"c580c86d-059f-4430-9137-52644b55bae4\"}" timestamp=1689546828663
Jul 17 03:34:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:53.563Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_write_seconds_total\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546820275
Jul 17 03:34:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:53.568Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"msh-tsgx12\", asset_id=\"113\", cpu=\"64\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546823391
Jul 17 03:34:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:53.612Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_packets_dropped_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"cadvisor\", interface=\"ens6f1np1\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546828905
Jul 17 03:34:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:53.641Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_info\", address=\"b8:ce:f6:91:ac:4c\", asset=\"msh-tsgx08\", asset_id=\"65\", broadcast=\"ff:ff:ff:ff:ff:ff\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens1f0np0\", duplex=\"full\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operstate=\"up\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546828328
Jul 17 03:34:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:53.668Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_timex_estimated_error_seconds\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546828328
Jul 17 03:34:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:53.719Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-fc23d3da\\\\x2d0a48\\\\x2d4a3e\\\\x2d9462\\\\x2d9171f119c4d2-volumes-kubernetes.io\\\\x7esecret-cm\\\\x2dcache\\\\x2dextra\\\\x2dsecrets.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689546831179
Jul 17 03:34:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:53.846Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.grpc.v1.cri-sandboxes-593349d6519f1b7aacc9df54cfa9a12b813bb4a6ae45ab570c5f3f65b7d503c1-shm.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689546831179
Jul 17 03:34:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:53.846Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db180\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546819367
Jul 17 03:34:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:53.861Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_info\", asset=\"msh-tsgx12\", asset_id=\"113\", container=\"monit-stream\", container_id=\"containerd://78ca9daf630ca7d94b44dbab8a8866633dd025fdea59e5e2b2dc1d67f3700ecf\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", image=\"registry.gdnt-cloud.website/mrzcpd-exporter:1.0.1\", image_id=\"sha256:b1a7adfb61be24490a1321c37405d86204b78b1ee251ed959d48dc357310fb1c\", image_spec=\"registry.gdnt-cloud.website/mrzcpd-exporter:1.0.1\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"mrzcpd-exporter-5cf7f967cc-2f76c\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"21526a39-5626-4c13-bf88-da6b44f8743f\"}" timestamp=1689546828663
Jul 17 03:34:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:53.862Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:80-0000:80:01.1-0000:81:00.0-host0-target0:2:0-0:2:0:0-block-sda.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689546831179
Jul 17 03:34:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:53.864Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cpu=\"35\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546819409
Jul 17 03:34:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:53.879Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"105\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546819409
Jul 17 03:34:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:53.909Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2dpartlabel-ONIE\\\\x2dBOOT.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689546825496
Jul 17 03:34:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:53.930Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_net_dev_group\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"cnibr0\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546829337
Jul 17 03:34:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:53.952Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Udp_Link_Double_ALL\", app_name=\"sapp4\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", vsysId=\"1\"}" timestamp=1689546820470
Jul 17 03:34:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:53.987Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:00-0000:00:01.1-0000:01:00.1-infiniband-mlx5_1.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689546831179
Jul 17 03:34:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:54.001Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"tmpfs\", name=\"data-var-lib-kubelet-pods-b049c496\\\\x2db9de\\\\x2d40a8\\\\x2d8a1b\\\\x2d4a23997ed556-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2dp4nvf.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689546831179
Jul 17 03:34:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:54.020Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_processes\", asset=\"msh-tsgx12\", asset_id=\"113\", container=\"firewall\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689546812704
Jul 17 03:34:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:54.074Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"phy_rx_bits_total\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", dev=\"ens1f1np1\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"mrzcpd-exporter\"}" timestamp=1689546824559
Jul 17 03:34:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:54.093Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-4832029edfe2512afd7c667f07ba975d766964db02fc0eb1e8d68fedfdb9eafa-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689546831179
Jul 17 03:34:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:54.405Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx08\", asset_id=\"65\", cpu=\"92\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546828328
Jul 17 03:34:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:54.420Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_drop_total\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"lo\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546828328
Jul 17 03:34:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:54.438Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_max_usage_bytes\", asset=\"msh-tsgx12\", asset_id=\"113\", container=\"traefik\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546818192
Jul 17 03:34:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:54.464Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_namespace_labels\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546828663
Jul 17 03:34:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:54.508Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2duuid-3579a7a8\\\\x2d6d85\\\\x2d4af2\\\\x2daaef\\\\x2d91ce88ecee46.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689546832135
Jul 17 03:34:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:54.528Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_errs_total\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth5a5e6ccb\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546823391
Jul 17 03:34:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:54.547Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_cpu_seconds_total\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-pod8ba148c1_f25f_4fa4_82b3_03b9d5a8ace6.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"slice\"}" timestamp=1689546831179
Jul 17 03:34:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:54.589Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:a0-0000:a0:03.1-0000:a1:00.1-infiniband_mad-umad7.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689546831179
Jul 17 03:34:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:54.653Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_down_changes_total\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth068249f9\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546823391
Jul 17 03:34:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:54.684Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_drop_total\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens1f1np1\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546823391
Jul 17 03:34:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:54.705Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_ready\", asset=\"msh-tsgx08\", asset_id=\"65\", container=\"prometheus-pushgateway\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-pushgateway-bd476bcf9-h4x26\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"f5340e71-ce09-4741-86dd-e1c191caf00d\"}" timestamp=1689546828950
Jul 17 03:34:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:54.727Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_address_assign_type\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth3449d361\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546828328
Jul 17 03:34:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:54.751Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"machine_nvm_capacity\", asset=\"pcap-tsgx07\", asset_id=\"85\", boot_id=\"b3f96875-e1d7-41b6-9294-5da2a36028b6\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"cadvisor\", machine_id=\"13844f7a1f364dbcb8e244533b708194\", mode=\"memory_mode\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", system_uuid=\"4c4c4544-005a-3210-8038-b3c04f5a4333\"}" timestamp=1689546830608
Jul 17 03:34:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:54.817Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db222\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546825184
Jul 17 03:34:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:54.832Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_max_fds\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"gpm.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689546820884
Jul 17 03:34:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:54.847Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_memory_used_rss_bytes\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546825184
Jul 17 03:34:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:54.948Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_start_time_seconds\", asset=\"pcap-tsgx07\", asset_id=\"85\", container=\"ipmi-exporter\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546830608
Jul 17 03:34:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:54.999Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-update-utmp.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689546820884
Jul 17 03:34:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:55.034Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db236\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546825184
Jul 17 03:34:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:55.089Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_netstat_Udp_RcvbufErrors\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546819409
Jul 17 03:34:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:56.456Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"55\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546825883
Jul 17 03:34:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:57.345Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"93\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546825883
Jul 17 03:34:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:57.434Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_fully_labeled_replicas\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-0-firewall-85f8dfd5cc\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546833389
Jul 17 03:34:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:57.452Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"69\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546825883
Jul 17 03:34:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:57.551Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"msh-dellr740-dt003\", asset_id=\"9\", cpu=\"39\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-dt003\", endpoint_id=\"229\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"hbase_master,clickhouse_data,kafka,gohangout,zookeeper,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546837410
Jul 17 03:34:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:57.564Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_vmstat_pgpgout\", asset=\"msh-dellr740-dt003\", asset_id=\"9\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-dt003\", endpoint_id=\"229\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"hbase_master,clickhouse_data,kafka,gohangout,zookeeper,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546837410
Jul 17 03:34:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:57.715Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_namespace_status_phase\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-public\", nz_agent_id=\"3\", phase=\"Active\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546833389
Jul 17 03:34:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:57.784Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_running\", asset=\"pcap-tsgx03\", asset_id=\"73\", container=\"kube-state-metrics\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-kube-state-metrics-6b768c7bf8-vxfl8\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"430b3649-17ca-4774-bc44-7d174d975598\"}" timestamp=1689546833389
Jul 17 03:34:58 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:58.024Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_spec_replicas\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"traefik\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546833389
Jul 17 03:34:58 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:58.040Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", pod=\"kubernetes-dashboard-66dd5b456-kl6gh\", project=\"TSG-OS\", project_id=\"7\", reason=\"Evicted\", service=\"prometheus-kube-state-metrics\", uid=\"95e7c99f-2ebd-42b8-abe6-583158385ffe\"}" timestamp=1689546833389
Jul 17 03:34:58 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:58.464Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"ck_merges_count\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"OLAP-Node-Exporter-pcap-dellr740-dt001\", endpoint_id=\"251\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546837557
Jul 17 03:34:58 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:34:58.619Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_hwmon_sensor_label\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", chip=\"platform_coretemp_0\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"OLAP-Node-Exporter-pcap-dellr740-dt001\", endpoint_id=\"251\", label=\"core_11\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\", sensor=\"temp13\"}" timestamp=1689546837557
Jul 17 03:35:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:01.112Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_info\", asset=\"msh-tsgx01\", asset_id=\"51\", container=\"prometheus-node-exporter\", container_id=\"containerd://de01079c24f3d95865413dd94c5846a0ecea818ecc47cb5362a016602f2dfcb0\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", image=\"quay.io/prometheus/node-exporter:v1.3.1\", image_id=\"sha256:1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26\", image_spec=\"quay.io/prometheus/node-exporter:v1.3.1\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-node-exporter-t6jf2\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"21204ec2-df19-472e-9ae1-06104ab28d13\"}" timestamp=1689546827425
Jul 17 03:35:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:01.247Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_ready\", asset=\"msh-tsgx01\", asset_id=\"51\", container=\"prometheus-server-configmap-reload\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-server-6ff784d758-x2kns\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"7e3e09eb-7bd8-47c8-bd0d-9b8a588f34f3\"}" timestamp=1689546827425
Jul 17 03:35:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:01.280Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-75913adc181c78326dfbd01e1344099a83546f181e1f0ed9bf66a24209fdf2fc-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689546824879
Jul 17 03:35:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:01.455Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_running\", asset=\"msh-tsgx01\", asset_id=\"51\", container=\"monit-stream\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"mrzcpd-exporter-5cf7f967cc-qcxqj\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"cfc14401-c454-4aca-ae14-7e249e5a7dc0\"}" timestamp=1689546827425
Jul 17 03:35:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:01.495Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_address_not_ready\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_endpoint=\"shaping-announce-port-tsg-traffic-engine-numa-1\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546827425
Jul 17 03:35:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:01.514Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_created\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_endpoint=\"dashboard-metrics-scraper\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546827425
Jul 17 03:35:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:01.529Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"1\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546829083
Jul 17 03:35:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:02.688Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_ready\", asset=\"msh-tsgx02\", asset_id=\"63\", container=\"traefik\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"traefik-64f55bb67d-lpmmw\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"11efef4a-05e0-44ee-8681-0e306eccb51c\"}" timestamp=1689546831412
Jul 17 03:35:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:02.722Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_address_assign_type\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens7f0np0\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546838318
Jul 17 03:35:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:02.767Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"tmpfs\", name=\"var-lib-kubelet-pods-67fd6de1\\\\x2dff70\\\\x2d485a\\\\x2da410\\\\x2d30979744acd0-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2dvkrgv.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689546828447
Jul 17 03:35:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:02.949Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"var-lib-kubelet-pods-87913277\\\\x2d0e7d\\\\x2d4545\\\\x2d8517\\\\x2d0068668d7130-volumes-kubernetes.io\\\\x7esecret-kubernetes\\\\x2ddashboard\\\\x2dcerts.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689546828447
Jul 17 03:35:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:03.151Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"msh-tsgx02\", asset_id=\"63\", container=\"telegraf-security\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", state=\"running\"}" timestamp=1689546819089
Jul 17 03:35:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:03.181Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_gc_duration_seconds\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", quantile=\"0.5\", service=\"ipmi-exporter\"}" timestamp=1689546834732
Jul 17 03:35:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:03.214Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx02\", asset_id=\"63\", cpu=\"70\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546838318
Jul 17 03:35:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:03.256Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx02\", asset_id=\"63\", cpu=\"29\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546838318
Jul 17 03:35:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:03.294Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx02\", asset_id=\"63\", cpu=\"9\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546838318
Jul 17 03:35:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:03.587Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_address_not_ready\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_endpoint=\"ipmi-exporter\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546831412
Jul 17 03:35:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:03.604Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"msh-tsgx02\", asset_id=\"63\", condition=\"false\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"mrzcpd-exporter-5cf7f967cc-9zrnz\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"0b1be0e3-e71c-4f3b-83a7-1d29910105b8\"}" timestamp=1689546831412
Jul 17 03:35:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:03.623Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_changes_total\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens2f1np1\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546838318
Jul 17 03:35:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:03.623Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx02\", asset_id=\"63\", cpu=\"18\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546838318
Jul 17 03:35:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:03.772Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"tmpfs\", name=\"run-k3s-containerd-io.containerd.grpc.v1.cri-sandboxes-f3e32b6b99348bfd481a4b517930a34254f7d5dd8ed8a945bc14711be6aae797-shm.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689546828447
Jul 17 03:35:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:03.867Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_info\", address=\"ba:8e:a2:f5:4c:f1\", asset=\"msh-tsgx02\", asset_id=\"63\", broadcast=\"ff:ff:ff:ff:ff:ff\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth421cdf56\", duplex=\"full\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operstate=\"up\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546838318
Jul 17 03:35:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:04.022Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"promhttp_metric_handler_requests_in_flight\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-dt001\", endpoint_id=\"225\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546842987
Jul 17 03:35:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:04.133Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_scaling_frequency_max_hertz\", asset=\"msh-dellr740-olap005\", asset_id=\"19\", cpu=\"21\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Spark-msh-dellr740-olap005\", endpoint_id=\"17\", module=\"NC-Spark\", module_id=\"47\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546843391
Jul 17 03:35:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:04.311Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"net_conntrack_dialer_conn_failed_total\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", dialer_name=\"425\", instance=\"10.10.20.159:10090\", reason=\"refused\", type=\"global\"}" timestamp=1689546842990
Jul 17 03:35:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:07.200Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_net_dev_group\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens7f0np0\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546843910
Jul 17 03:35:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:07.261Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_carrier_total\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"cnibr0\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546843910
Jul 17 03:35:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:07.278Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cpu=\"110\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546843910
Jul 17 03:35:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:07.342Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db177\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546838290
Jul 17 03:35:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:07.388Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cpu=\"47\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546843910
Jul 17 03:35:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:07.394Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_packets_total\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens1f1np1\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546843910
Jul 17 03:35:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:07.423Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_name_assign_type\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth2d0ed10f\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546843910
Jul 17 03:35:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:07.480Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_max_fds\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"smartd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689546833582
Jul 17 03:35:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:07.484Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"cortex_kv_request_duration_seconds_bucket\", agent_name=\"msh-kvm-nz001\", application=\"cortex\", datacenter=\"MSH\", instance=\"127.0.0.1:10093\", kv_name=\"ingester-ring\", le=\"0.1\", operation=\"GET\", role=\"primary\", status_code=\"200\", type=\"inmemory\"}" timestamp=1689546847261
Jul 17 03:35:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:07.529Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_annotations\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"prometheus-alertmanager-6df7f457b6\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546841227
Jul 17 03:35:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:07.733Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"cortex_kv_request_duration_seconds_bucket\", agent_name=\"msh-kvm-nz001\", application=\"cortex\", datacenter=\"MSH\", instance=\"127.0.0.1:10093\", kv_name=\"store-gateway\", le=\"5.0\", operation=\"GET\", role=\"primary\", status_code=\"200\", type=\"inmemory\"}" timestamp=1689546847261
Jul 17 03:35:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:07.862Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-b1acab13fb36088033f79ef87a8a5031bacd125779e7bc737ad4b5e58dc2a4a4.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"scope\"}" timestamp=1689546833582
Jul 17 03:35:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:07.921Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-infiniband-umad10.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689546833582
Jul 17 03:35:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:07.998Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_ready\", asset=\"twa-tsgx01\", asset_id=\"91\", container=\"firewall\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", pod=\"tsg-traffic-engine-numa-0-firewall-584cc6bdff-5rl2w\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"cd272aa8-aacf-4f0d-971f-5dbf27e21826\"}" timestamp=1689546841227
Jul 17 03:35:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:08.110Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_dormant\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens8f0np0\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546843910
Jul 17 03:35:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:08.280Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_scrape_collector_duration_seconds\", asset=\"twa-tsgx01\", asset_id=\"91\", collector=\"schedstat\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546843910
Jul 17 03:35:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:10.451Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_STATS_metric_deleteBatchTime_99_9th_percentile\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546848804
Jul 17 03:35:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:10.489Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_compactionsCompletedCount\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"2442970a6ce71a8e1e1ae56690e800ff\", table=\"knowledge_base_hos_bucket\", type=\"data\"}" timestamp=1689546848804
Jul 17 03:35:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:10.791Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"jmx_config_reload_failure_total\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", module=\"DC-HBase\", module_id=\"3\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546848804
Jul 17 03:35:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:10.923Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_getCount\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"firewall_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"335727b6b83d9b052005b2861958972d\", table=\"index_time_firewall_hos_bucket\", type=\"index\"}" timestamp=1689546848804
Jul 17 03:35:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:11.269Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_compactionsFailedCount\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"rtp_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"b796339c8a4f4ca7bac1f51f208df0e5\", table=\"index_time_rtp_hos_bucket\", type=\"index\"}" timestamp=1689546848804
Jul 17 03:35:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:11.430Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_194_3_metric_majorCompactionInputFileCount_min\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546850147
Jul 17 03:35:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:11.579Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_3_metric_majorCompactionOutputFileCount_mean\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546850147
Jul 17 03:35:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:11.705Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_session_record_hos_bucket_metric_deleteBatchTime_98th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546850147
Jul 17 03:35:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:11.727Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_193_1_metric_compactedInputBytes\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546850147
Jul 17 03:35:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:11.771Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_194_1_metric_majorCompactionOutputFileCount_99th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546850147
Jul 17 03:35:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:11.855Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_tsg_galaxy_table_relation_framedip_account_metric_putTime_99_9th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546850147
Jul 17 03:35:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:12.276Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_3_metric_compactionInputFileCount_num_ops\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546850147
Jul 17 03:35:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:12.318Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_memStoreSize\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"rtp_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"f7f1f4fc67718d48d80d0634e0df3169\", table=\"index_time_rtp_hos_bucket\", type=\"index\"}" timestamp=1689546850147
Jul 17 03:35:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:12.376Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_DecreaseMemStoreSize_num_ops\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Memory\"}" timestamp=1689546850147
Jul 17 03:35:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:12.378Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_194_1_metric_scanSize_25th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546850147
Jul 17 03:35:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:12.416Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_tsg_table_report_result_metric_scanTime_99_9th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546850147
Jul 17 03:35:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:12.498Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_193_1_metric_majorCompactionInputSize_min\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546850147
Jul 17 03:35:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:12.511Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_replicaid\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"268471f048bc85ce60683d08da793a10\", table=\"assessment_hos_bucket\", type=\"data\"}" timestamp=1689546850147
Jul 17 03:35:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:15.231Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"117\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546841230
Jul 17 03:35:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:15.290Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_ulimits_soft\", asset=\"twa-tsgx07\", asset_id=\"103\", container=\"mrapm-device\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", ulimit=\"max_open_files\"}" timestamp=1689546832511
Jul 17 03:35:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:15.296Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dnf-makecache.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689546847046
Jul 17 03:35:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:15.424Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_reservation_limit_bytes\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689546849150
Jul 17 03:35:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:15.447Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"47\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546841230
Jul 17 03:35:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:15.468Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"metrics-server\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546848076
Jul 17 03:35:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:15.631Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_start_time_seconds\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546849150
Jul 17 03:35:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:15.655Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"113\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546841230
Jul 17 03:35:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:15.669Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"112\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546841230
Jul 17 03:35:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:15.713Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:40-0000:40:01.1-0000:41:00.0-net-ens3f0np0.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689546847046
Jul 17 03:35:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:15.783Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"twa-tsgx07\", asset_id=\"103\", container=\"prometheus-alertmanager\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"cadvisor\", failure_type=\"pgmajfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"container\"}" timestamp=1689546845932
Jul 17 03:35:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:15.793Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-ens6f0np0.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689546847046
Jul 17 03:35:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:15.799Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_daemonset_created\", asset=\"twa-tsgx07\", asset_id=\"103\", daemonset=\"dign-server\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546848076
Jul 17 03:35:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:15.820Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_hwmon_temp_celsius\", asset=\"twa-tsgx07\", asset_id=\"103\", chip=\"0000:80:01_2_0000:82:00_0\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", sensor=\"temp1\", service=\"prometheus-node-exporter\"}" timestamp=1689546841230
Jul 17 03:35:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:15.835Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_resident_memory_bytes\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"getty@tty1.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689546847046
Jul 17 03:35:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:15.858Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_spec_replicas\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-0-firewall-f8d7c6884\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546848076
Jul 17 03:35:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:16.010Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"scrape_duration_seconds\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_instance=\"10.10.30.167:9100\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546841230
Jul 17 03:35:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:16.056Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"7\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546841230
Jul 17 03:35:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:16.245Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"16\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546841230
Jul 17 03:35:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:16.269Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_packets_dropped_total\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"cadvisor\", interface=\"ens3f0np0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546847993
Jul 17 03:35:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:16.299Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_annotations\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-1-firewall-6b6bf4679c\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546848076
Jul 17 03:35:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:16.614Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546841230
Jul 17 03:35:17 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:17.222Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546841230
Jul 17 03:35:17 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:17.276Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_metadata_resource_version\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.tsg-traffic-engine-numa-1.v2\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546848076
Jul 17 03:35:18 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:18.497Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"clickhouse_replicated_fetch\", asset=\"twa-dellr740-dt003\", asset_id=\"41\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"NC-Clickhouse-twa-dellr740-dt003\", endpoint_id=\"91\", module=\"NC-Clickhouse\", module_id=\"29\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546858300
Jul 17 03:35:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:19.569Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failcnt\", asset=\"twa-tsgx06\", asset_id=\"101\", container=\"dign-client\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546848673
Jul 17 03:35:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:19.628Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_lookups_total\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"ipmi-exporter\"}" timestamp=1689546852714
Jul 17 03:35:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:19.668Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_down_changes_total\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth9d00126e\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546857612
Jul 17 03:35:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:19.710Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"90\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546857612
Jul 17 03:35:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:19.984Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"twa-tsgx06\", asset_id=\"101\", container=\"prometheus-alertmanager-configmap-reload\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"cadvisor\", failure_type=\"pgfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"container\"}" timestamp=1689546848681
Jul 17 03:35:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:20.369Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_queue_length\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens8f0np0\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546857612
Jul 17 03:35:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:20.461Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_fifo_total\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth9421f8b7\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546857612
Jul 17 03:35:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:20.499Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"nsfs\", name=\"run-netns-cni\\\\x2debdb49b6\\\\x2d0344\\\\x2d4c95\\\\x2dbef7\\\\x2d8efd759dd088.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689546844356
Jul 17 03:35:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:20.566Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"11\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546848263
Jul 17 03:35:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:20.587Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_frame_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"vethd6f33409\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546848263
Jul 17 03:35:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:20.692Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_processes\", asset=\"pcap-tsgx10\", asset_id=\"89\", container=\"lb-tcp-80\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546838535
Jul 17 03:35:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:20.714Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_packets_total\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"cadvisor\", interface=\"ens4f2np2\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546844340
Jul 17 03:35:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:20.722Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_annotations\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.traefik.v1\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546853828
Jul 17 03:35:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:20.739Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-e7995082a6a36bced4eaf18ccd3a8ac666bc7ffdf79e960c48916536e50da455-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689546850657
Jul 17 03:35:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:20.767Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"11\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546848263
Jul 17 03:35:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:20.785Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:80-0000:80:01.1-0000:81:00.0-host0-target0:2:0-0:2:0:0-block-sda-sda1.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689546850657
Jul 17 03:35:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:20.907Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_drop_total\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethef8c731a\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546857612
Jul 17 03:35:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:20.934Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-remount-fs.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689546850657
Jul 17 03:35:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:20.959Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"98\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546848263
Jul 17 03:35:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:20.973Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-remount-fs.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689546844356
Jul 17 03:35:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:20.985Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_state_id\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"mlx5_9\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546848263
Jul 17 03:35:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:21.018Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_frame_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth36380b9e\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546848263
Jul 17 03:35:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:21.089Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"30\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546857612
Jul 17 03:35:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:21.111Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_compressed_total\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens4f2np2\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546857612
Jul 17 03:35:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:21.163Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"phy_rx_missed_total\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", dev=\"ens1f0np0\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"mrzcpd-exporter\"}" timestamp=1689546857986
Jul 17 03:35:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:21.242Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_reservation_limit_bytes\", asset=\"twa-tsgx06\", asset_id=\"101\", container=\"prometheus-server-configmap-reload\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546852894
Jul 17 03:35:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:21.257Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"127\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546857612
Jul 17 03:35:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:21.570Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_bytes_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"vethd5440e08\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546848263
Jul 17 03:35:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:21.826Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Unknown\", pod=\"helm-install-tsg-traffic-engine-numa-0-d2b2s\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"7c729184-3a15-4a27-8e65-311777e49481\"}" timestamp=1689546853828
Jul 17 03:35:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:21.879Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_bytes_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens8f1np1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546848263
Jul 17 03:35:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:21.893Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"phy_rx_missed_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", dev=\"ens7f1np1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"mrzcpd-exporter\"}" timestamp=1689546854036
Jul 17 03:35:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:22.053Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_start_time_seconds\", asset=\"pcap-tsgx10\", asset_id=\"89\", container=\"telegraf-security\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689546855735
Jul 17 03:35:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:23.537Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_netstat_Tcp_RetransSegs\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546854790
Jul 17 03:35:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:23.697Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db86\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546857019
Jul 17 03:35:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:23.749Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_ready\", asset=\"msh-tsgx09\", asset_id=\"55\", container=\"kube-state-metrics\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-kube-state-metrics-6b768c7bf8-db9cd\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"1c96629c-a0c6-4e4c-8e97-b6bbe91aa6d9\"}" timestamp=1689546860068
Jul 17 03:35:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:23.780Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_restarts_total\", asset=\"msh-tsgx09\", asset_id=\"55\", container=\"dign-server-dns\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"dign-server-pd9mz\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"4c92ea3a-4155-49d3-9d36-59027e09d717\"}" timestamp=1689546860068
Jul 17 03:35:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:23.797Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_annotations\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.traefik-crd.v1\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546860068
Jul 17 03:35:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:23.968Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_errs_total\", asset=\"msh-dellr740-olap004\", asset_id=\"17\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"em1\", endpoint=\"NC-Spark-msh-dellr740-olap004\", endpoint_id=\"15\", module=\"NC-Spark\", module_id=\"47\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546862253
Jul 17 03:35:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:24.027Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_timex_frequency_adjustment_ratio\", asset=\"msh-dellr740-olap004\", asset_id=\"17\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Spark-msh-dellr740-olap004\", endpoint_id=\"15\", module=\"NC-Spark\", module_id=\"47\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546862253
Jul 17 03:35:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:24.172Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_speed_bytes\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"vethc0962f36\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546860160
Jul 17 03:35:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:24.179Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"prometheus-node-exporter\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"uninterruptible\"}" timestamp=1689546855504
Jul 17 03:35:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:24.227Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db198\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546861795
Jul 17 03:35:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:24.248Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-sda3.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689546851102
Jul 17 03:35:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:24.344Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_metadata_resource_version\", asset=\"pcap-tsgx12\", asset_id=\"117\", configmap=\"kube-root-ca.crt\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546851122
Jul 17 03:35:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:24.428Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_init_container_status_waiting\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"tsg-dign-init\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"dign-server-bhs24\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"78d32c4f-349d-455c-932b-b75d141ec6e9\"}" timestamp=1689546851122
Jul 17 03:35:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:24.453Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_heap_idle_bytes\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"ipmi-exporter\"}" timestamp=1689546854325
Jul 17 03:35:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:24.491Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_free_bytes\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"shm\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/781481587068effa36560b73e2c1b985686c2beb9bdbdf0b4e89b3a6277aabc5/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546860160
Jul 17 03:35:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:24.602Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"rdma-ndd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689546858416
Jul 17 03:35:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:24.617Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_compressed_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens7f1np1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546860160
Jul 17 03:35:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:24.636Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_drop_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth7ba955f4\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546860160
Jul 17 03:35:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:24.760Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_memory_used_overhead_bytes\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546861795
Jul 17 03:35:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:24.775Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.grpc.v1.cri-sandboxes-9278e24d65ca2abe9945e75398d615ce969e9aab821b6c0ba4062251bd8e1a07-shm.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689546851102
Jul 17 03:35:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:24.816Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_usage_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"prometheus-server\", cpu=\"total\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546845500
Jul 17 03:35:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:24.904Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"24\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546860160
Jul 17 03:35:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:24.923Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"62\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546860160
Jul 17 03:35:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:25.007Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-8ea47e8a\\\\x2dbb11\\\\x2d4592\\\\x2daee9\\\\x2d0458e6cbc258-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2dfx279.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689546851102
Jul 17 03:35:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:25.945Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-cc35e1619f0fa3f112f8701176209000b1b0820394226210cc97a8063d73e448-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689546853907
Jul 17 03:35:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:26.047Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_ulimits_soft\", asset=\"twa-tsgx09\", asset_id=\"107\", container=\"lb-tcp-443\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", ulimit=\"max_open_files\"}" timestamp=1689546848436
Jul 17 03:35:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:26.056Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"twa-tsgx09\", asset_id=\"107\", condition=\"unknown\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", pod=\"tsg-traffic-engine-numa-0-firewall-d55dcdb59-dgxgd\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"48a4a0ca-42d3-4487-bc89-7615c4f62e66\"}" timestamp=1689546858744
Jul 17 03:35:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:26.057Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"115\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546855810
Jul 17 03:35:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:26.083Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_files\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"shm\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/3d23979fb1c874f13c17f63e6650ac026e1b85e98520be72f3e36f71f3406389/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546855810
Jul 17 03:35:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:26.114Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Tcp_Close_byK\", app_name=\"sapp4\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", vsysId=\"1\"}" timestamp=1689546851010
Jul 17 03:35:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:26.176Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"108\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546855810
Jul 17 03:35:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:26.286Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_edac_csrow_correctable_errors_total\", asset=\"twa-tsgx09\", asset_id=\"107\", controller=\"1\", csrow=\"3\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546855810
Jul 17 03:35:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:26.345Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"61\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546855810
Jul 17 03:35:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:26.383Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"rpc-statd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689546853907
Jul 17 03:35:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:27.189Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_swap\", asset=\"twa-tsgx09\", asset_id=\"107\", container=\"prometheus-server-configmap-reload\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546856231
Jul 17 03:35:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:27.406Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"dashboard-metrics-scraper\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", failure_type=\"pgfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"hierarchy\"}" timestamp=1689546858956
Jul 17 03:35:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:27.411Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"88\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546862882
Jul 17 03:35:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:27.475Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"89\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546862882
Jul 17 03:35:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:27.514Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_receive_switch_relay_errors_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_2\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546862882
Jul 17 03:35:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:27.885Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_annotations\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-traefik-crd-m2nsh\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"6ee7b6d8-27b2-4118-9cf0-9229465d0b5b\"}" timestamp=1689546855682
Jul 17 03:35:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:27.890Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_down_changes_total\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens4f3np3\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546855810
Jul 17 03:35:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:27.903Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_last_seen\", asset=\"twa-tsgx09\", asset_id=\"107\", container=\"telegraf-security\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689546865520
Jul 17 03:35:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:27.926Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_annotations\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_endpoint=\"prometheus-pushgateway\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546858744
Jul 17 03:35:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:27.929Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"var-lib-kubelet-pods-848571bc\\\\x2da31e\\\\x2d44de\\\\x2da0f0\\\\x2dd172cf8f3bd0-volume\\\\x2dsubpaths-telegraf-telegraf-0.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689546854114
Jul 17 03:35:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:27.945Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_dormant\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"eno35np2\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546855810
Jul 17 03:35:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:27.964Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"15\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546855810
Jul 17 03:35:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:28.059Z caller=dedupe.go:112 component=remote level=error remote_name=52db6f url=http://10.10.10.159:10090/nz-agent/prometheus/proxy/api/v1/write msg="non-recoverable error" count=1000 exemplarCount=0 err="server returned HTTP status 400 : out of order sample"
Jul 17 03:35:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:28.100Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-dd88c173\\\\x2d8b8c\\\\x2d4cdc\\\\x2dafaf\\\\x2db521eb296cc1-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2dc76dv.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689546854114
Jul 17 03:35:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:28.190Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dbus.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689546854114
Jul 17 03:35:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:28.204Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_bytes_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"virtio_dign_s\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546862882
Jul 17 03:35:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:28.221Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db139\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546859446
Jul 17 03:35:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:28.402Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_annotations\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"traefik-64f55bb67d-r4jmw\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"fde9828e-69b9-41c0-b67a-234d7bed9f69\"}" timestamp=1689546855682
Jul 17 03:35:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:28.417Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_transmit_wait_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_7\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546862882
Jul 17 03:35:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:28.425Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"60\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546862882
Jul 17 03:35:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:28.443Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_up_changes_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens2f1np1\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546862882
Jul 17 03:35:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:28.464Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_flags\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"cnibr0\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546862882
Jul 17 03:35:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:28.476Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-7b13c51d7b16133af270753c151c606633bb81b2611d763e8fbcfbe8be30a5ac-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689546854114
Jul 17 03:35:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:28.618Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Succeeded\", pod=\"traefik-64f55bb67d-r4jmw\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"fde9828e-69b9-41c0-b67a-234d7bed9f69\"}" timestamp=1689546855682
Jul 17 03:35:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:28.665Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_up_changes_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"br_dign_c\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546862882
Jul 17 03:35:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:28.825Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_files\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"shm\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/6315b6edc2b1c8011a08c2423868cd963dbde3eae84c0a788d98c57be05bbc48/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546862882
Jul 17 03:35:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:28.843Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_up\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens6f1np1\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546862882
Jul 17 03:35:30 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:30.987Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"105\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546860939
Jul 17 03:35:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:31.003Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-pod4079ea2e_97aa_4a21_888a_7befd9a893de.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"slice\"}" timestamp=1689546861015
Jul 17 03:35:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:31.147Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"74\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546860939
Jul 17 03:35:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:31.647Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"52\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546860939
Jul 17 03:35:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:31.659Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_frame_total\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"bond0\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546859093
Jul 17 03:35:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:31.663Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_annotations\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-0-firewall-69747b4bf9\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546870436
Jul 17 03:35:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:31.680Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_fifo_total\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"cnibr0\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546860939
Jul 17 03:35:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:31.690Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_up_changes_total\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens7f0np0\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546859093
Jul 17 03:35:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:31.702Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db26\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546870141
Jul 17 03:35:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:31.726Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_fifo_total\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth4c8f7059\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546859093
Jul 17 03:35:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:31.734Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"56\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546860791
Jul 17 03:35:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:31.786Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx03\", asset_id=\"95\", container=\"kube-multus\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Total\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546848738
Jul 17 03:35:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:31.800Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"twa-tsgx03\", asset_id=\"95\", container=\"prometheus-pushgateway\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"stopped\"}" timestamp=1689546853183
Jul 17 03:35:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:31.813Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dracut-pre-udev.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"service\"}" timestamp=1689546861015
Jul 17 03:35:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:31.815Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_entropy_pool_size_bits\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546860791
Jul 17 03:35:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:31.821Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"25\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546859093
Jul 17 03:35:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:31.850Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_metadata_generation\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"tsg-traffic-engine-numa-1-firewall\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546864068
Jul 17 03:35:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:31.941Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"msh-tsgx11\", asset_id=\"111\", condition=\"true\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"svclb-traefik-46c02faf-rzf44\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"59e0f471-ccc6-4ff5-8571-7fb136f39b65\"}" timestamp=1689546859276
Jul 17 03:35:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:31.971Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"abrt-journal-core.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"service\"}" timestamp=1689546862644
Jul 17 03:35:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:32.011Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_rate_bytes_per_second\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546860791
Jul 17 03:35:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:32.029Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"27\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546860791
Jul 17 03:35:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:32.204Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"twa-tsgx03\", asset_id=\"95\", container=\"exporter-merger\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"cadvisor\", failure_type=\"pgfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"hierarchy\"}" timestamp=1689546858263
Jul 17 03:35:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:32.213Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_errs_total\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens4f2np2\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546860791
Jul 17 03:35:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:32.224Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Running\", pod=\"local-path-provisioner-957fdf8bc-hv6cn\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"73c566ef-6d83-4dd3-9f97-e8eda48fa60f\"}" timestamp=1689546864068
Jul 17 03:35:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:32.229Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"95\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546860791
Jul 17 03:35:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:32.239Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:a0-0000:a0:03.1-0000:a1:00.0-net-ens6f0np0.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689546861015
Jul 17 03:35:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:32.277Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx05\", asset_id=\"67\", container=\"coredns\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Discard\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546850484
Jul 17 03:35:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:32.295Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"110\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546860791
Jul 17 03:35:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:32.498Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_created\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"cm-cache-extra-secrets\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546859894
Jul 17 03:35:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:32.642Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link_mode\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"eno2\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546860791
Jul 17 03:35:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:32.658Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:80-0000:80:01.2-0000:82:00.0-net-ens4f0np0.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689546862644
Jul 17 03:35:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:32.736Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"mcelog.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"service\"}" timestamp=1689546866907
Jul 17 03:35:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:32.769Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"ftx_pkts_total\", app=\"sapp4-1\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"nf_1_fw\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"mrzcpd-exporter\", serviceFunctionIndex=\"1\"}" timestamp=1689546863017
Jul 17 03:35:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:32.784Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2did-scsi\\\\x2dSDELL_PERC_H345_Adp_002c663bc430ea252c00598284f0a7ce\\\\x2dpart4.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689546867691
Jul 17 03:35:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:32.836Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-pode02c929a_3c07_4477_b04a_8ffc16c405f5.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"slice\"}" timestamp=1689546867691
Jul 17 03:35:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:33.119Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_packets_total\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth9a152f90\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546860791
Jul 17 03:35:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:33.124Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546859093
Jul 17 03:35:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:33.201Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"82\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546859093
Jul 17 03:35:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:33.290Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"firewall\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", state=\"running\"}" timestamp=1689546849303
Jul 17 03:35:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:33.359Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_labels\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-kube-state-metrics-6b768c7bf8-h2rv4\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"37526140-01d6-46f5-9f0b-e066a6254bac\"}" timestamp=1689546859894
Jul 17 03:35:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:33.383Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_observed_generation\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-1-firewall-74d974c88c\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546859894
Jul 17 03:35:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:33.409Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"twa-tsgx05\", asset_id=\"99\", condition=\"Progressing\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"metrics-server\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"true\"}" timestamp=1689546859894
Jul 17 03:35:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:33.532Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_hwmon_temp_celsius\", asset=\"twa-tsgx05\", asset_id=\"99\", chip=\"0000:60:03_1_0000:63:00_1\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", sensor=\"temp1\", service=\"prometheus-node-exporter\"}" timestamp=1689546860605
Jul 17 03:35:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:33.535Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_info\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", job_name=\"helm-install-cm-policy-local-cache\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546859894
Jul 17 03:35:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:33.558Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_created\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-0-firewall-5bc67584dc\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546859894
Jul 17 03:35:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:33.619Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_cpu_period\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"firewall\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689546862009
Jul 17 03:35:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:33.631Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-virtual-net-vethc5e0d95c.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689546862644
Jul 17 03:35:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:33.647Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"overlay\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-39c97aa000da243bc35e391bae98d9ab1504aa4d220b3370b4e89916f125b31f-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689546867691
Jul 17 03:35:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:33.666Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_start_time\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-systemd-exporter-f9mp6\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"fa64de1f-70a4-425a-9810-f9f7d5175ae0\"}" timestamp=1689546859276
Jul 17 03:35:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:33.816Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"21\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546860605
Jul 17 03:35:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:33.845Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"ipmi-exporter\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"sleeping\"}" timestamp=1689546853978
Jul 17 03:35:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:33.885Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_ready\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"prometheus-server-configmap-reload\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-server-6ff784d758-mv9fn\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"461d1588-9ea2-41cf-8eca-daa78ce6d555\"}" timestamp=1689546859276
Jul 17 03:35:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:34.001Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_cpu_period\", asset=\"twa-tsgx05\", asset_id=\"99\", container=\"telegraf\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689546858535
Jul 17 03:35:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:34.020Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_persistentvolumeclaim_info\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", persistentvolumeclaim=\"prometheus-server\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", storageclass=\"local-path\", volumename=\"pvc-c5689dcb-14fe-4a93-8001-73849d4955f4\"}" timestamp=1689546859894
Jul 17 03:35:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:34.167Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db255\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546869260
Jul 17 03:35:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:34.183Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"plymouth-switch-root.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"service\"}" timestamp=1689546867691
Jul 17 03:35:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:34.325Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_writes_bytes_total\", asset=\"twa-tsgx05\", asset_id=\"99\", container=\"lb-tcp-443\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546838473
Jul 17 03:35:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:34.346Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"119\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546860605
Jul 17 03:35:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:34.360Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Udp_Link_S2C_ALL\", app_name=\"sapp4\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", vsysId=\"1\"}" timestamp=1689546863815
Jul 17 03:35:38 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:38.369Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_filteredReadRequestCount\", asset=\"twa-dellr740-dt003\", asset_id=\"41\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt003-HregionServer\", endpoint_id=\"163\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"1\", parent=\"default\", project=\"OLAP\", project_id=\"3\", region=\"dad4a709f7bbc7ff52bbf40af074d986\", table=\"index_partfile_default\", type=\"index\"}" timestamp=1689546876625
Jul 17 03:35:39 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:39.172Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_gc_duration_seconds_count\", asset=\"msh-kvm-cm001\", asset_id=\"3\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"CM-Branch-Redis-msh-kvm-cm001\", endpoint_id=\"319\", host=\"192.168.20.157\", module=\"CM-Branch-Redis\", module_id=\"73\", nz_agent_id=\"3\", parent_asset=\"msh-dellr740-pve001\", parent_asset_id=\"1\", project=\"Central-Management\", project_id=\"5\"}" timestamp=1689546878579
Jul 17 03:35:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:40.468Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_namespace_status_phase\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-node-lease\", nz_agent_id=\"3\", phase=\"Active\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546880034
Jul 17 03:35:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:40.518Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"96\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546871647
Jul 17 03:35:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:40.535Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"98\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546871647
Jul 17 03:35:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:40.612Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_flags\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens8f1np1\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546871647
Jul 17 03:35:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:40.655Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_free_bytes\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"shm\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/e9aa25a33698d95841aab03c5f1d5842f70bead06e8836bd9c0af9d422de0175/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546871647
Jul 17 03:35:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:40.705Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_node_status_condition\", asset=\"twa-tsgx04\", asset_id=\"97\", condition=\"Ready\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", node=\"localhost\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"true\"}" timestamp=1689546880034
Jul 17 03:35:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:40.753Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_owner\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", owner_is_controller=\"true\", owner_kind=\"Deployment\", owner_name=\"tsg-traffic-engine-numa-0-firewall\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-0-firewall-7cbb5bf99c\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546880034
Jul 17 03:35:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:40.771Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"10\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546871647
Jul 17 03:35:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:40.776Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"72\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546871647
Jul 17 03:35:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:40.796Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_flags\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vetha33ccb73\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546871647
Jul 17 03:35:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:40.807Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"88\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546871647
Jul 17 03:35:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:40.821Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_free_bytes\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"shm\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/6bb3c4b01c7dc060834c9ac3dcfbdb1b98953169a67a177f842dcc26fca81c99/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546871647
Jul 17 03:35:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:40.837Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_compressed_total\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens1f1np1\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546871647
Jul 17 03:35:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:40.856Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db160\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546873827
Jul 17 03:35:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:40.874Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"39\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546871647
Jul 17 03:35:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:40.928Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_10_194_1_metric_majorCompactionInputFileCount_98th_percentile\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546879761
Jul 17 03:35:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:40.978Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_hbase_table_meta_metric_compactionInputFileCount_median\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546879761
Jul 17 03:35:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:41.007Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_memory_SReclaimable_bytes\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546871647
Jul 17 03:35:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:41.041Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"emergency.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"service\"}" timestamp=1689546875764
Jul 17 03:35:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:41.059Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_10_193_1_metric_compactionInputSize_median\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546879761
Jul 17 03:35:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:41.075Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_10_193_1_metric_putTime_max\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546879761
Jul 17 03:35:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:41.089Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_errs_total\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth01fab232\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546871647
Jul 17 03:35:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:41.107Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_fully_labeled_replicas\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"ipmi-exporter-6f8464dd94\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546880034
Jul 17 03:35:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:41.108Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_errs_total\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens8f1np1\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546871647
Jul 17 03:35:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:41.123Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"76\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546871647
Jul 17 03:35:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:41.146Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db221\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546873827
Jul 17 03:35:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:41.211Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"80\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546871647
Jul 17 03:35:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:41.227Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"104\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546871647
Jul 17 03:35:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:41.247Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_10_194_1_metric_appendTime_98th_percentile\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546879761
Jul 17 03:35:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:41.256Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_maxStoreFileAge\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"troubleshooting_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"91d4c2795c97291695318034adb40e90\", table=\"index_partfile_troubleshooting_hos_bucket\", type=\"index\"}" timestamp=1689546879761
Jul 17 03:35:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:41.292Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_CHILD_LINK_metric_putTime_90th_percentile\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546879761
Jul 17 03:35:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:41.393Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_deleteCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"bd87f0ce10bc02fc873c13816b123c65\", table=\"knowledge_base_hos_bucket\", type=\"data\"}" timestamp=1689546879761
Jul 17 03:35:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:41.415Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_deleteCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", namespace=\"SYSTEM\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"67ba66d072c3c5f11913412d93b085ff\", table=\"LOG\", type=\"data\"}" timestamp=1689546879761
Jul 17 03:35:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:41.496Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_packets_dropped_total\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"cadvisor\", interface=\"ens2f0np0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546865785
Jul 17 03:35:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:41.517Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_discards_transmitted_total\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_10\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546871647
Jul 17 03:35:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:41.533Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_transmit_wait_total\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_3\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546871647
Jul 17 03:35:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:41.891Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_address_available\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_endpoint=\"shaping-announce-port-tsg-traffic-engine-numa-1\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546880034
Jul 17 03:35:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:41.920Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_complete\", asset=\"twa-tsgx04\", asset_id=\"97\", condition=\"unknown\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", job_name=\"helm-install-ipmi-exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546880034
Jul 17 03:35:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:41.965Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-776e4314eab7328eca726ecdbce064935ce7541916115e2889c71376f5e95a1c.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"scope\"}" timestamp=1689546875764
Jul 17 03:35:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:41.989Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"var.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689546875764
Jul 17 03:35:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:42.202Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_LOG_metric_putTime_90th_percentile\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546879761
Jul 17 03:35:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:42.683Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_memStoreSize\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"session_record_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"bb66c7a6883ea68b08c121a94154bb7b\", table=\"index_partfile_session_record_hos_bucket\", type=\"index\"}" timestamp=1689546879761
Jul 17 03:35:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:43.446Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db247\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546874838
Jul 17 03:35:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:43.572Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_dormant\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens3f1np1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546876242
Jul 17 03:35:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:43.590Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_fifo_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"eno1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546876242
Jul 17 03:35:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:43.614Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Unknown_Pkt\", app_name=\"sapp4\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", vsysId=\"1\"}" timestamp=1689546867581
Jul 17 03:35:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:43.750Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_annotations\", asset=\"pcap-tsgx01\", asset_id=\"71\", configmap=\"coredns\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546875154
Jul 17 03:35:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:43.768Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas_unavailable\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"prometheus-pushgateway\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546875154
Jul 17 03:35:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:43.787Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", cpu=\"19\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546876242
Jul 17 03:35:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:43.864Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_symbol_error_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"mlx5_0\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546876242
Jul 17 03:35:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:43.904Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_drop_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens6f0np0\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546876242
Jul 17 03:35:44 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:44.228Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_created\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-systemd-exporter-cbljh\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"da14c78d-f512-4a0f-b73a-808e63c92307\"}" timestamp=1689546875154
Jul 17 03:35:44 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:44.241Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_net_dev_group\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth841d6801\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546876242
Jul 17 03:35:46 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:46.215Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_ResourceManager_running_1440\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-Yarn-pcap-dellr740-dt001-ResourceManager\", endpoint_id=\"135\", module=\"DC-Yarn\", module_id=\"1\", name=\"QueueMetrics\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", q0=\"root\"}" timestamp=1689546885217
Jul 17 03:35:48 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:48.765Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_default_metric_incrementTime_min\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546887438
Jul 17 03:35:48 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:48.766Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_3_metric_compactionOutputSize_99th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546887438
Jul 17 03:35:48 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:48.805Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_default_metric_compactionInputSize_95th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546887438
Jul 17 03:35:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:49.593Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_numFilesCompactedCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"proxy_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"4e3139889b447635cc39482c39741a58\", table=\"index_filename_proxy_hos_bucket\", type=\"index\"}" timestamp=1689546887438
Jul 17 03:35:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:49.694Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"twa-tsgx10\", asset_id=\"109\", cpu=\"120\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546881797
Jul 17 03:35:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:49.705Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_193_1_metric_appendTime_num_ops\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546887438
Jul 17 03:35:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:49.710Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_lastMajorCompactionAge\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"firewall_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"d697e1d2f8732a6c48600cdc04c12377\", table=\"index_time_firewall_hos_bucket\", type=\"index\"}" timestamp=1689546887438
Jul 17 03:35:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:49.718Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:00-0000:00:01.1-0000:01:00.1-net-ens1f1np1.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689546881057
Jul 17 03:35:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:49.752Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_4_metric_putTime_25th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546887438
Jul 17 03:35:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:49.773Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_hbase_table_namespace_metric_compactionOutputSize_99_9th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546887438
Jul 17 03:35:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:49.876Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_constraint_errors_received_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546886709
Jul 17 03:35:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:49.894Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_appendCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"firewall_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"312be68cb14448f0189460056d4d500f\", table=\"index_time_firewall_hos_bucket\", type=\"index\"}" timestamp=1689546887438
Jul 17 03:35:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:49.908Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"tsg-clixon-restconf.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689546881057
Jul 17 03:35:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:49.945Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-ad54bd72f5b9444d785b76bd8ef5097a1fe94e26bed92654eed781dbb206dd3e-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689546881057
Jul 17 03:35:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:49.990Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"58\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546886709
Jul 17 03:35:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:50.020Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_daemonset_status_number_available\", asset=\"twa-tsgx10\", asset_id=\"109\", daemonset=\"dign-server\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546879668
Jul 17 03:35:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:50.039Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_errs_total\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"cnibr0\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546881797
Jul 17 03:35:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:50.055Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"dashboard-metrics-scraper\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Total\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546870726
Jul 17 03:35:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:50.116Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_reads_merged_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"overlay_0-195\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546884254
Jul 17 03:35:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:50.143Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_limit_bytes\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/6e4f2f4687bae3bbca1a2a2cbbc5a63313dfd06a46f8ecb30e6b1173c86a1f59/shm\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546884254
Jul 17 03:35:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:50.149Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_writes_merged_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/f9018ed6bd85ccf87fa84a57ebe1e8f139adfc841aed3294bca3ee381fea5005/shm\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546884254
Jul 17 03:35:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:50.164Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"twa-tsgx08\", asset_id=\"105\", container=\"lb-tcp-80\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"uninterruptible\"}" timestamp=1689546879667
Jul 17 03:35:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:50.173Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_frequency_min_hertz\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"70\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546886709
Jul 17 03:35:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:50.190Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"virtio_dign_s\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546886709
Jul 17 03:35:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:50.206Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Send_Tcp_Rst\", app_name=\"sapp4\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", vsysId=\"1\"}" timestamp=1689546888675
Jul 17 03:35:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:50.223Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_down_changes_total\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethec314a7d\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546881797
Jul 17 03:35:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:50.228Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_speed_bytes\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethe417278f\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546881797
Jul 17 03:35:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:50.259Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"system-systemd\\\\x2dcoredump.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"slice\"}" timestamp=1689546881057
Jul 17 03:35:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:50.265Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Ipv4_Bit\", app_name=\"sapp4\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", vsysId=\"1\"}" timestamp=1689546882447
Jul 17 03:35:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:50.274Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_labels\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"shaping-announce-port-tsg-traffic-engine-numa-0\", uid=\"539315cf-d33e-41f7-8d0f-073fbca9e817\"}" timestamp=1689546879668
Jul 17 03:35:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:50.317Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_flushesQueuedCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"firewall_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"e726937d65b545da8ae97961134fc920\", table=\"index_filename_firewall_hos_bucket\", type=\"index\"}" timestamp=1689546887438
Jul 17 03:35:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:50.335Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2dpath-pci\\\\x2d0000:81:00.0\\\\x2dscsi\\\\x2d0:2:0:0\\\\x2dpart1.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689546881057
Jul 17 03:35:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:50.389Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_sockstat_UDP_inuse\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546886709
Jul 17 03:35:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:50.414Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-541bab6f5b69ec62a4bbbac81886b37f6ce5fdadef1cc6d1499fcf4f42a9b7f1-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689546881057
Jul 17 03:35:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:50.421Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx10\", asset_id=\"109\", cpu=\"0\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546881797
Jul 17 03:35:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:50.456Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_default_metric_filteredReadRequestCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546887438
Jul 17 03:35:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:50.468Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"83\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546886709
Jul 17 03:35:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:50.472Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"twa-tsgx10\", asset_id=\"109\", cpu=\"57\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546881797
Jul 17 03:35:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:50.476Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-7554ff525f8f143bb91e9ee7736d0a155c1a23948b078bd5970ca5988327ea9b-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689546881057
Jul 17 03:35:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:50.507Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_maxStoreFileAge\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"22a2444c5578382f2c9b2238d198336b\", table=\"default\", type=\"data\"}" timestamp=1689546887438
Jul 17 03:35:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:50.523Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-tmpfiles-setup-dev.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"service\"}" timestamp=1689546881057
Jul 17 03:35:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:50.533Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas_updated\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"prometheus-kube-state-metrics\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546881568
Jul 17 03:35:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:50.534Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_FsReadTime_90th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"IO\"}" timestamp=1689546887438
Jul 17 03:35:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:50.545Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_address_available\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_endpoint=\"prometheus-kube-state-metrics\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546881568
Jul 17 03:35:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:50.586Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"atd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"service\"}" timestamp=1689546886554
Jul 17 03:35:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:50.650Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_created\", asset=\"twa-tsgx10\", asset_id=\"109\", configmap=\"prometheus-server\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546879668
Jul 17 03:35:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:50.695Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"102\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546886709
Jul 17 03:35:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:50.777Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_node_status_condition\", asset=\"twa-tsgx08\", asset_id=\"105\", condition=\"DiskPressure\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", node=\"localhost\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"false\"}" timestamp=1689546881568
Jul 17 03:35:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:50.835Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_id\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethde236744\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546886709
Jul 17 03:35:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:50.849Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens6f1np1\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546886709
Jul 17 03:35:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:50.859Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"twa-tsgx10\", asset_id=\"109\", cpu=\"33\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546881797
Jul 17 03:35:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:50.867Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"84\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546886709
Jul 17 03:35:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:50.880Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-28e75c4d\\\\x2d4cd3\\\\x2d41cb\\\\x2d8435\\\\x2ddd13365e6e7d-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2drfqtb.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689546881057
Jul 17 03:35:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:50.888Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled_time\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-tsg-traffic-engine-numa-1-4qrh5\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"996d9871-ef2b-4f18-9376-ea4e633eb786\"}" timestamp=1689546879668
Jul 17 03:35:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:50.901Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:a0-0000:a0:03.1-0000:a1:00.0-net-ens6f0np0.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689546881057
Jul 17 03:35:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:50.908Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_mtu_bytes\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens1f1np1\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546881797
Jul 17 03:35:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:50.912Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_frequency_min_hertz\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546886709
Jul 17 03:35:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:50.926Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"twa-tsgx10\", asset_id=\"109\", cpu=\"43\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546881797
Jul 17 03:35:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:50.965Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx10\", asset_id=\"109\", cpu=\"9\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546881797
Jul 17 03:35:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:51.024Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_inodes_total\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"overlay_0-47\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546876055
Jul 17 03:35:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:51.054Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"dign-client\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Sync\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546871295
Jul 17 03:35:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:51.074Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_annotations\", asset=\"twa-tsgx10\", asset_id=\"109\", configmap=\"kube-root-ca.crt\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"consul\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546879668
Jul 17 03:35:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:51.089Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"11\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546886709
Jul 17 03:35:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:51.105Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"87\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546886709
Jul 17 03:35:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:51.144Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"import-state.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689546881057
Jul 17 03:35:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:51.157Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-1c6adf295e169e5435abc78195bc6d7e760d8521681d770d7d0e756a96e2a04e.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"scope\"}" timestamp=1689546881057
Jul 17 03:35:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:51.209Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_swap\", asset=\"twa-tsgx08\", asset_id=\"105\", container=\"prometheus-node-exporter\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546886149
Jul 17 03:35:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:51.385Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_reads_merged_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"overlay_0-424\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546884254
Jul 17 03:35:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:51.388Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_annotations\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-1-firewall-6b6bf4679c\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546881568
Jul 17 03:35:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:51.391Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_running\", asset=\"twa-tsgx08\", asset_id=\"105\", container=\"dign-server-dns\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"dign-server-zj2bv\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"d4457315-9a37-45e8-b7b6-d04b15cea130\"}" timestamp=1689546881568
Jul 17 03:35:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:51.406Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_created\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-ipmi-exporter-pspfb\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"8d60fd58-fc4a-4fec-a27c-e595a24480b5\"}" timestamp=1689546881568
Jul 17 03:35:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:52.444Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_symbol_error_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"mlx5_0\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546889337
Jul 17 03:35:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:52.753Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_compressed_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"eno1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546889337
Jul 17 03:35:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:52.798Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", cpu=\"43\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546889337
Jul 17 03:35:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:52.833Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-1f0a2164d250c7fdd3f11e65c867e2c9a6b0e002537de1f226bd6cf95c9c2557.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"scope\"}" timestamp=1689546885496
Jul 17 03:35:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:53.011Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_scaling_frequency_min_hertz\", asset=\"pcap-dellr740-dt002\", asset_id=\"33\", cpu=\"39\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"NC-Gohangout-pcap-dellr740-dt002\", endpoint_id=\"173\", module=\"DC-Gohangout\", module_id=\"35\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546892670
Jul 17 03:35:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:53.143Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"ftx_pkts_total\", app=\"sapp4-1\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"nf_1_fw\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"mrzcpd-exporter\", serviceFunctionIndex=\"1\"}" timestamp=1689546888118
Jul 17 03:35:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:53.157Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_type\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"chart-values-cm-policy-local-cache\", service=\"prometheus-kube-state-metrics\", type=\"Opaque\"}" timestamp=1689546884478
Jul 17 03:35:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:53.165Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_labels\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"localhost.node-password.k3s\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546884478
Jul 17 03:35:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:53.185Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_bytes_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens6f0np0\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546889337
Jul 17 03:35:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:53.195Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_sockstat_FRAG_memory\", asset=\"pcap-dellr740-dt002\", asset_id=\"33\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"NC-Gohangout-pcap-dellr740-dt002\", endpoint_id=\"173\", module=\"DC-Gohangout\", module_id=\"35\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546892670
Jul 17 03:35:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:53.211Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"pcap-dellr740-dt002\", asset_id=\"33\", cpu=\"22\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"NC-Gohangout-pcap-dellr740-dt002\", endpoint_id=\"173\", module=\"DC-Gohangout\", module_id=\"35\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546892670
Jul 17 03:35:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:53.550Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_flags\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth2dc20da0\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546889337
Jul 17 03:35:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:53.553Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:20-0000:20:03.1-0000:21:00.0-infiniband-mlx5_2.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689546885496
Jul 17 03:35:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:53.567Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-pod9726a0d7_a3a3_4c4e_8e73_81e501ab0b51.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"slice\"}" timestamp=1689546885496
Jul 17 03:35:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:53.610Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_link_error_recovery_total\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_0\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546883391
Jul 17 03:35:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:53.676Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", cpu=\"88\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546889337
Jul 17 03:35:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:53.690Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"eno35np2\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546879409
Jul 17 03:35:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:53.747Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cpu=\"100\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546879409
Jul 17 03:35:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:53.784Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db98\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546885184
Jul 17 03:35:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:53.791Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_bytes_total\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"lo\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546888328
Jul 17 03:35:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:53.805Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_errs_total\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"lo\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546888328
Jul 17 03:35:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:53.813Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx12\", asset_id=\"113\", cpu=\"94\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546883391
Jul 17 03:35:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:53.891Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", container=\"dign-server-dns\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Sync\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546882869
Jul 17 03:35:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:53.894Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_protocol_type\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"eno36np3\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546879409
Jul 17 03:35:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:53.902Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_spec_replicas\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"mrzcpd-exporter\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546888950
Jul 17 03:35:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:53.906Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"prometheus-alertmanager\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546888950
Jul 17 03:35:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:53.921Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"msh-tsgx08\", asset_id=\"65\", cpu=\"124\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546888328
Jul 17 03:35:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:53.930Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"msh-tsgx08\", asset_id=\"65\", container=\"prometheus-pushgateway\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"cadvisor\", failure_type=\"pgfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"container\"}" timestamp=1689546877065
Jul 17 03:35:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:53.960Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Failed\", pod=\"kube-multus-ds-thbtz\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"fe410bd7-7bfd-4526-8256-38727d23f01b\"}" timestamp=1689546882058
Jul 17 03:35:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:53.996Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cpu=\"20\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546879409
Jul 17 03:35:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:54.018Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_io_current\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"overlay_0-47\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546880428
Jul 17 03:35:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:54.020Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx12\", asset_id=\"113\", cpu=\"85\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546883391
Jul 17 03:35:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:54.033Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-pod3a6dd972_267f_421e_975e_5b26c18a80ad.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"slice\"}" timestamp=1689546880884
Jul 17 03:35:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:54.037Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_created\", asset=\"pcap-tsgx11\", asset_id=\"115\", configmap=\"kube-root-ca.crt\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546884478
Jul 17 03:35:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:54.166Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"msh-tsgx08\", asset_id=\"65\", container=\"mrapm-device\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"cadvisor\", failure_type=\"pgfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"container\"}" timestamp=1689546869756
Jul 17 03:35:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:54.168Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_constraint_errors_transmitted_total\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_5\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546883391
Jul 17 03:35:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:54.182Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"tmpfs\", name=\"data-var-lib-kubelet-pods-c03a604a\\\\x2d11b7\\\\x2d419d\\\\x2d889f\\\\x2df6fba056b2fa-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2dqvjqd.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689546892135
Jul 17 03:35:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:54.186Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_errs_total\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"vethcd3875c5\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546883391
Jul 17 03:35:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:54.197Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_mod_info\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"github.com/alecthomas/units\", nz_agent_id=\"3\", program=\"systemd_exporter\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", version=\"v0.0.0-20190924025748-f65c72e2690d\"}" timestamp=1689546892135
Jul 17 03:35:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:54.209Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_swap\", asset=\"msh-tsgx12\", asset_id=\"113\", container=\"telegraf\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689546877865
Jul 17 03:35:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:54.223Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"process_virtual_memory_max_bytes\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546883391
Jul 17 03:35:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:54.262Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx12\", asset_id=\"113\", cpu=\"49\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546883391
Jul 17 03:35:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:54.346Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_swap_limit_bytes\", asset=\"pcap-tsgx07\", asset_id=\"85\", container=\"ipmi-exporter\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546890608
Jul 17 03:35:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:54.477Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx12\", asset_id=\"113\", cpu=\"46\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546883391
Jul 17 03:35:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:54.593Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"iscsi-onboot.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689546880884
Jul 17 03:35:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:54.595Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_swap_limit_bytes\", asset=\"pcap-tsgx07\", asset_id=\"85\", container=\"firewall\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689546890608
Jul 17 03:35:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:54.654Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx12\", asset_id=\"113\", cpu=\"54\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546883391
Jul 17 03:35:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:54.803Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2duuid-eb3ef54c\\\\x2de946\\\\x2d46ea\\\\x2db82c\\\\x2d3ff03e40f719.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689546880884
Jul 17 03:35:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:54.821Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"mnt-onie\\\\x2dboot.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689546880884
Jul 17 03:35:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:54.983Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cpu=\"42\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546879409
Jul 17 03:35:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:55.037Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cpu=\"100\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546879409
Jul 17 03:35:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:55.396Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cpu=\"27\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546879409
Jul 17 03:35:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:55.429Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"unbound-anchor.timer\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"timer\"}" timestamp=1689546880884
Jul 17 03:35:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:56.308Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens8f1np1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546885883
Jul 17 03:35:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:56.348Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db105\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546886475
Jul 17 03:35:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:56.412Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"tsg-clixon-backend.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"simple\", type=\"service\"}" timestamp=1689546883372
Jul 17 03:35:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:56.429Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_fifo_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth88dfc5f3\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546885883
Jul 17 03:35:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:56.574Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Tcp_Bit\", app_name=\"sapp4\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", vsysId=\"1\"}" timestamp=1689546890331
Jul 17 03:35:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:56.595Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-c7c3f554bb5564ecf3c0b65d130e40b6beaced6e6ad0c45c1efcc258a66f6217.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"scope\"}" timestamp=1689546883372
Jul 17 03:35:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:56.607Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db178\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546886475
Jul 17 03:35:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:56.705Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_daemonset_status_observed_generation\", asset=\"pcap-tsgx03\", asset_id=\"73\", daemonset=\"prometheus-node-exporter\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546893389
Jul 17 03:35:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:56.908Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_commands_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cmd=\"rename\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546886475
Jul 17 03:35:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:56.913Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"clickhouse_os_soft_irq_time_cpu44\", asset=\"msh-dellr740-olap001\", asset_id=\"11\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Clickhouse-msh-dellr740-olap001\", endpoint_id=\"71\", module=\"NC-Clickhouse\", module_id=\"29\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546895798
Jul 17 03:35:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:56.922Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db124\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546886475
Jul 17 03:35:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:56.936Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"shm\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/12893bbc4b1f51657a188cea56ff12e411716f2f83f5ce99e8acac1b22cce414/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546885883
Jul 17 03:35:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:56.959Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_packets_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens6f0np0\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546885883
Jul 17 03:35:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:56.964Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_timer_last_trigger_seconds\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-tmpfiles-clean.timer\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689546883372
Jul 17 03:35:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:57.133Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"pcap-tsgx03\", asset_id=\"73\", condition=\"Progressing\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"systemd-exporter\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"unknown\"}" timestamp=1689546893389
Jul 17 03:35:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:57.196Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Failed\", pod=\"coredns-f5d5fbf4d-d7dz4\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"06d9f977-06a7-4069-895f-e4ebce64d667\"}" timestamp=1689546893389
Jul 17 03:35:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:57.213Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", phase=\"Failed\", pod=\"dashboard-metrics-scraper-5b94897dc9-pkp4g\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"99474c52-8652-4755-b917-62a7ae1a3d6c\"}" timestamp=1689546893389
Jul 17 03:35:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:57.236Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"23\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546885883
Jul 17 03:35:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:57.320Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_packets_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"cadvisor\", interface=\"virtio_dign_c\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546887437
Jul 17 03:35:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:57.334Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"pcap-tsgx03\", asset_id=\"73\", condition=\"Progressing\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"prometheus-kube-state-metrics\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"unknown\"}" timestamp=1689546893389
Jul 17 03:35:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:57.353Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_fully_labeled_replicas\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-1-firewall-5474c977\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546893389
Jul 17 03:35:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:57.372Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"40\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546885883
Jul 17 03:35:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:57.384Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2did-scsi\\\\x2d362cea7f0848257002c2c7c71e1fae165\\\\x2dpart2.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689546883372
Jul 17 03:35:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:57.426Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_edac_csrow_correctable_errors_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", controller=\"0\", csrow=\"3\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546885883
Jul 17 03:35:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:57.441Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_memory_DirectMap2M_bytes\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546885883
Jul 17 03:35:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:57.957Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", cpu=\"27\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"OLAP-Node-Exporter-pcap-dellr740-dt001\", endpoint_id=\"251\", mode=\"nice\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546897557
Jul 17 03:35:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:59.707Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_type\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"chart-values-prometheus\", service=\"prometheus-kube-state-metrics\", type=\"Opaque\"}" timestamp=1689546887425
Jul 17 03:35:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:59.722Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_dormant\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"cnibr0\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546889083
Jul 17 03:35:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:59.759Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"74\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546889083
Jul 17 03:35:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:35:59.965Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"dign-server-t2gqw\", project=\"TSG-OS\", project_id=\"7\", reason=\"Shutdown\", service=\"prometheus-kube-state-metrics\", uid=\"c6ffcf61-3641-43ac-b76c-b24b78ac0cd3\"}" timestamp=1689546887425
Jul 17 03:36:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:00.266Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_edac_csrow_correctable_errors_total\", asset=\"msh-tsgx01\", asset_id=\"51\", controller=\"0\", csrow=\"unknown\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546889083
Jul 17 03:36:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:00.276Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"101\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546889083
Jul 17 03:36:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:00.290Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"126\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546889083
Jul 17 03:36:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:00.336Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_mspan_inuse_bytes\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546889083
Jul 17 03:36:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:00.353Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_speed_bytes\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens7f0np0\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546889083
Jul 17 03:36:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:00.365Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"21\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546889083
Jul 17 03:36:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:00.418Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_cpu_shares\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689546894593
Jul 17 03:36:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:00.492Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_address_available\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_endpoint=\"shaping-announce-port-tsg-traffic-engine-numa-1\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546887425
Jul 17 03:36:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:00.536Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_waiting\", asset=\"msh-tsgx01\", asset_id=\"51\", container=\"dign-server-dns\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"dign-server-t2gqw\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"c6ffcf61-3641-43ac-b76c-b24b78ac0cd3\"}" timestamp=1689546887425
Jul 17 03:36:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:00.556Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"1\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546889083
Jul 17 03:36:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:00.565Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"27\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546889083
Jul 17 03:36:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:00.585Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_constraint_errors_transmitted_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_2\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546889083
Jul 17 03:36:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:00.600Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"88\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546889083
Jul 17 03:36:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:00.754Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"32\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546889083
Jul 17 03:36:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:00.769Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db76\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546885071
Jul 17 03:36:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:00.786Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_reads_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/shm\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546892294
Jul 17 03:36:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:00.803Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-548b900871a45d57101cfb436aec0a3fcaab374253e3af668927e0d3ac20467a.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"scope\"}" timestamp=1689546884879
Jul 17 03:36:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:00.884Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_labels\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-mrzcpd-exporter-6xctw\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"64e78458-5407-4009-8438-11206cb5f43d\"}" timestamp=1689546887425
Jul 17 03:36:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:02.625Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_state_started\", asset=\"msh-tsgx02\", asset_id=\"63\", container=\"helm\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-ipmi-exporter-mqzww\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"d3f976a5-a42f-4d79-8f22-dcd41aaf53f6\"}" timestamp=1689546891412
Jul 17 03:36:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:02.644Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda5\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", fstype=\"ext4\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/data/var/lib/kubelet/pods/43d98779-0eca-4764-ab37-a5e20cb923be/volume-subpaths/telegraf/telegraf-security/0\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546898318
Jul 17 03:36:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:02.666Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"msh-tsgx02\", asset_id=\"63\", cpu=\"115\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546898318
Jul 17 03:36:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:02.670Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"msh-tsgx02\", asset_id=\"63\", container=\"prometheus-alertmanager\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"running\"}" timestamp=1689546884647
Jul 17 03:36:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:02.713Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_frees_total\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"ipmi-exporter\"}" timestamp=1689546894732
Jul 17 03:36:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:03.082Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_fifo_total\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth14e44624\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546898318
Jul 17 03:36:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:03.139Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"msh-tsgx02\", asset_id=\"63\", cpu=\"99\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546898318
Jul 17 03:36:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:03.156Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dbus.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689546888447
Jul 17 03:36:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:03.187Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"lldpad.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"service\"}" timestamp=1689546888447
Jul 17 03:36:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:03.205Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_metadata_resource_version\", asset=\"msh-tsgx02\", asset_id=\"63\", configmap=\"telegraf-tsg-traffic-engine-numa-0\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546891412
Jul 17 03:36:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:03.218Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db10\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546889269
Jul 17 03:36:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:03.237Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"net_conntrack_dialer_conn_failed_total\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", dialer_name=\"asset_ping_67\", instance=\"10.10.20.159:10090\", reason=\"refused\", type=\"global\"}" timestamp=1689546902993
Jul 17 03:36:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:03.238Z caller=dedupe.go:112 component=remote level=info remote_name=52db6f url=http://10.10.10.159:10090/nz-agent/prometheus/proxy/api/v1/write msg="Dropped sample for series that was not explicitly dropped via relabelling" ref=4436650
Jul 17 03:36:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:03.260Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"prometheus_http_request_duration_seconds_bucket\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", handler=\"/api/v1/series\", instance=\"10.10.20.159:10090\", le=\"8\", type=\"global\"}" timestamp=1689546902993
Jul 17 03:36:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:03.293Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", cpu=\"3\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-dt001\", endpoint_id=\"225\", mode=\"nice\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546902992
Jul 17 03:36:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:03.400Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"net_conntrack_dialer_conn_attempted_total\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", dialer_name=\"49\", instance=\"10.10.20.159:10090\", type=\"global\"}" timestamp=1689546902993
Jul 17 03:36:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:03.514Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_free_bytes\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sdb1\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-dt001\", endpoint_id=\"225\", fstype=\"xfs\", module=\"OLAP-Node-Exporter\", module_id=\"51\", mountpoint=\"/home\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546902992
Jul 17 03:36:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:03.819Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"net_conntrack_dialer_conn_failed_total\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", dialer_name=\"241\", instance=\"10.10.20.159:10090\", reason=\"timeout\", type=\"global\"}" timestamp=1689546902993
Jul 17 03:36:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:03.856Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"prometheus_http_request_duration_seconds_bucket\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", handler=\"/-/healthy\", instance=\"10.10.20.159:10090\", le=\"1\", type=\"global\"}" timestamp=1689546902993
Jul 17 03:36:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:04.213Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_memory_AnonHugePages_bytes\", asset=\"msh-dellr740-olap005\", asset_id=\"19\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Spark-msh-dellr740-olap005\", endpoint_id=\"17\", module=\"NC-Spark\", module_id=\"47\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546903392
Jul 17 03:36:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:04.264Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_netstat_Udp_InErrors\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-dt001\", endpoint_id=\"225\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546902992
Jul 17 03:36:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:06.704Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx01\", asset_id=\"91\", container=\"dign-client\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Total\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546886743
Jul 17 03:36:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:06.825Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_ips\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", ip=\"10.42.0.167\", ip_family=\"4\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"traefik-64f55bb67d-bd6jb\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"377e59e7-1c6e-433a-a40c-92bbd7c96064\"}" timestamp=1689546901227
Jul 17 03:36:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:06.893Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cpu=\"79\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546903910
Jul 17 03:36:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:06.954Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cpu=\"18\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546903910
Jul 17 03:36:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:06.956Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_packets_total\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"cadvisor\", interface=\"virtio_dign_s\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546901231
Jul 17 03:36:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:06.975Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_compressed_total\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"br_dign_c\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546903910
Jul 17 03:36:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:07.077Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_max_fds\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"rsyslog.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689546893582
Jul 17 03:36:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:07.219Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_spec_replicas\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-0-firewall-6bd8dcb747\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546901227
Jul 17 03:36:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:07.460Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"twa-tsgx01\", asset_id=\"91\", container=\"kube-state-metrics\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"cadvisor\", failure_type=\"pgmajfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"hierarchy\"}" timestamp=1689546900575
Jul 17 03:36:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:07.473Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cpu=\"32\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546903910
Jul 17 03:36:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:07.496Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-rdma-devices-mlx5_5:1-umad.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689546893582
Jul 17 03:36:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:07.540Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_compressed_total\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens4f2np2\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546903910
Jul 17 03:36:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:07.981Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"100\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546903910
Jul 17 03:36:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:08.009Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cpu=\"24\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546903910
Jul 17 03:36:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:08.025Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_metadata_resource_version\", asset=\"twa-tsgx01\", asset_id=\"91\", configmap=\"cm-policy-local-cache-redis-configuration\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546901227
Jul 17 03:36:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:08.054Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_virtual_memory_bytes\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"gpm.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689546893582
Jul 17 03:36:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:08.055Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cpu=\"98\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546903910
Jul 17 03:36:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:08.096Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"rpc-statd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"service\"}" timestamp=1689546893582
Jul 17 03:36:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:08.112Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_scrape_collector_duration_seconds\", asset=\"twa-tsgx01\", asset_id=\"91\", collector=\"ipvs\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546903910
Jul 17 03:36:09 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:09.618Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_compactionsQueuedCount\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"default\", project=\"OLAP\", project_id=\"3\", region=\"61ecde15b0197c24043c6c1d3fa7957d\", table=\"index_time_default\", type=\"index\"}" timestamp=1689546908804
Jul 17 03:36:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:10.275Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_STATS_metric_compactionTime_98th_percentile\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546908804
Jul 17 03:36:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:10.390Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_compactionsFailedCount\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"e14b6811437ae968954ef3ada3528474\", table=\"firewall_hos_bucket\", type=\"data\"}" timestamp=1689546908804
Jul 17 03:36:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:10.438Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_minStoreFileAge\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"rtp_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"da4d07ac6e142d510f271e8e5a276908\", table=\"index_filename_rtp_hos_bucket\", type=\"index\"}" timestamp=1689546908804
Jul 17 03:36:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:10.539Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"java_lang_GarbageCollector_LastGcInfo_memoryUsageAfterGc_committed\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", key=\"Code Cache\", module=\"DC-HBase\", module_id=\"3\", name=\"ParNew\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546908804
Jul 17 03:36:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:10.553Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_compactionsQueuedCount\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"troubleshooting_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"bf2a458b5ee504849c8f27ed46a87a35\", table=\"index_filename_troubleshooting_hos_bucket\", type=\"index\"}" timestamp=1689546908804
Jul 17 03:36:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:11.368Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_PostIndexUpdateTime_SYSTEM:MUTEX_median\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"PhoenixIndexer\"}" timestamp=1689546910150
Jul 17 03:36:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:11.390Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_DeleteBatch_99th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Server\"}" timestamp=1689546910150
Jul 17 03:36:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:11.554Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_1_metric_majorCompactionOutputSize_num_ops\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546910150
Jul 17 03:36:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:12.088Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_3_metric_majorCompactionOutputSize_90th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546910150
Jul 17 03:36:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:12.143Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_maxFlushQueueSize\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"rtp_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"85fabb55b1e307fd3aed990df32c42db\", table=\"index_filename_rtp_hos_bucket\", type=\"index\"}" timestamp=1689546910150
Jul 17 03:36:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:12.156Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_LOG_metric_appendTime_75th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546910150
Jul 17 03:36:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:12.236Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_compressed_total\", asset=\"twa-dellr740-dt002\", asset_id=\"39\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"em2\", endpoint=\"OLAP-Node-Exporter-twa-dellr740-dt002\", endpoint_id=\"259\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546912055
Jul 17 03:36:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:12.237Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_putCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"knowledge_base_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"d7545eddd6684b532e594f03edaf2b47\", table=\"index_time_knowledge_base_hos_bucket\", type=\"index\"}" timestamp=1689546910150
Jul 17 03:36:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:12.350Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_2_metric_flushMemstoreSize_75th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546910150
Jul 17 03:36:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:12.408Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_compactionsCompletedCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"knowledge_base_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"408298ac4baa4d15f68738efc36b7451\", table=\"index_filename_knowledge_base_hos_bucket\", type=\"index\"}" timestamp=1689546910150
Jul 17 03:36:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:12.413Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_1_metric_compactionInputSize_75th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546910150
Jul 17 03:36:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:12.423Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_deleteCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"0cff77a0a9217551821dd184adc37cf2\", table=\"assessment_hos_bucket\", type=\"data\"}" timestamp=1689546910150
Jul 17 03:36:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:12.631Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_1_metric_majorCompactionInputFileCount_98th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546910150
Jul 17 03:36:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:12.834Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_193_1_metric_flushOutputSize_25th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546910150
Jul 17 03:36:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:12.837Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_tsg_galaxy_table_relation_framedip_account_metric_putTime_max\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546910150
Jul 17 03:36:13 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:13.011Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_2_metric_flushOutputSize_95th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546910150
Jul 17 03:36:13 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:13.231Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_193_1_metric_compactionOutputSize_25th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546910150
Jul 17 03:36:13 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:13.274Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_2_metric_putTime_median\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546910150
Jul 17 03:36:13 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:13.304Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_2_metric_storeFileSize\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546910150
Jul 17 03:36:13 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:13.421Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_FUNCTION_metric_deleteTime_99_9th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546910150
Jul 17 03:36:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:15.723Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-731f805f\\\\x2dbea9\\\\x2d4ef2\\\\x2da05b\\\\x2d3bdad1e50dd2-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2djg7h4.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689546907046
Jul 17 03:36:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:16.039Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_status_succeeded\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", job_name=\"helm-install-tsg-traffic-engine-numa-0\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546908076
Jul 17 03:36:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:16.086Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_created\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.prometheus.v1\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546908076
Jul 17 03:36:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:16.168Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_last_seen\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689546909181
Jul 17 03:36:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:16.182Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_processes\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689546908164
Jul 17 03:36:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:16.741Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_queue_length\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth48e28e0d\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546901230
Jul 17 03:36:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:16.768Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_mapped_file\", asset=\"twa-tsgx07\", asset_id=\"103\", container=\"traefik\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546906322
Jul 17 03:36:17 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:17.526Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_last_seen\", asset=\"twa-tsgx07\", asset_id=\"103\", container=\"local-path-provisioner\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546909168
Jul 17 03:36:17 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:17.721Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"16\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546901230
Jul 17 03:36:17 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:17.985Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"initrd-cleanup.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"service\"}" timestamp=1689546907046
Jul 17 03:36:18 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:18.034Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_swap\", asset=\"twa-tsgx07\", asset_id=\"103\", container=\"prometheus-server\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546907443
Jul 17 03:36:18 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:18.077Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_errors_total\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"cadvisor\", interface=\"ens1f0np0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546904031
Jul 17 03:36:18 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:18.170Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_unicast_packets_transmitted_total\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_6\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546901230
Jul 17 03:36:18 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:18.377Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"clickhouse_network_receive_errors_em3\", asset=\"twa-dellr740-dt003\", asset_id=\"41\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"NC-Clickhouse-twa-dellr740-dt003\", endpoint_id=\"91\", module=\"NC-Clickhouse\", module_id=\"29\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546918300
Jul 17 03:36:18 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:18.445Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"1\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546901230
Jul 17 03:36:18 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:18.482Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"26\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546901230
Jul 17 03:36:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:19.544Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_created\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", job_name=\"helm-install-traefik-crd\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546913949
Jul 17 03:36:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:19.871Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"12\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546917612
Jul 17 03:36:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:20.052Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_frame_total\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"eno34np1\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546917612
Jul 17 03:36:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:20.216Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_address_not_ready\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_endpoint=\"shaping-announce-port-tsg-traffic-engine-numa-1\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546913949
Jul 17 03:36:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:20.253Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"abrt-vmcore.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"oneshot\", type=\"service\"}" timestamp=1689546904356
Jul 17 03:36:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:20.270Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2dlabel-TGS\\\\x2dOS\\\\x2dUPDATE.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689546904356
Jul 17 03:36:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:20.496Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_timex_tai_offset_seconds\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546908263
Jul 17 03:36:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:20.513Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_observed_generation\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"systemd-exporter\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546913828
Jul 17 03:36:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:20.533Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"microcode.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689546910657
Jul 17 03:36:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:20.539Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_ready_replicas\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"kubernetes-dashboard-66dd5b456\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546913828
Jul 17 03:36:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:20.842Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_created\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"cm-policy-local-cache-redis-master\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546913828
Jul 17 03:36:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:20.938Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_packets_total\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"cadvisor\", interface=\"eth0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689546894050
Jul 17 03:36:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:21.122Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_observed_generation\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-1-firewall-86f6f8b98d\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546913828
Jul 17 03:36:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:21.233Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"pcap-tsgx10\", asset_id=\"89\", condition=\"Available\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"tsg-traffic-engine-numa-1-firewall\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"true\"}" timestamp=1689546913828
Jul 17 03:36:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:21.418Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_annotations\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-ipmi-exporter-ntwmm\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"6d124c79-91fd-435c-a730-222ebe3b4194\"}" timestamp=1689546913828
Jul 17 03:36:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:21.429Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"twa-tsgx06\", asset_id=\"101\", condition=\"true\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-configmap-reloader-pmpzk\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"86d10a1f-8af3-4a7c-b9f6-a63e3e9ca659\"}" timestamp=1689546913949
Jul 17 03:36:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:21.433Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_readonly\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"shm\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/219012e52aedd3297447f89162fe8b51fe5cc898f7483af85df1b649a58054f9/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546908263
Jul 17 03:36:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:21.470Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"14\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546917612
Jul 17 03:36:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:21.485Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-coredump@0.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"service\"}" timestamp=1689546904356
Jul 17 03:36:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:21.531Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_unicast_packets_received_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"mlx5_8\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546908263
Jul 17 03:36:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:21.914Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_net_dev_group\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"vethdf3e6cb0\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546908263
Jul 17 03:36:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:21.973Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_mtu_bytes\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"lo\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546908263
Jul 17 03:36:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:22.038Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_speed_bytes\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"vetha7f11396\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546908263
Jul 17 03:36:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:22.101Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_scrape_collector_duration_seconds\", asset=\"pcap-tsgx10\", asset_id=\"89\", collector=\"hwmon\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546908263
Jul 17 03:36:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:22.362Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db159\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546914840
Jul 17 03:36:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:22.527Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db181\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546914840
Jul 17 03:36:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:22.571Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db41\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546914840
Jul 17 03:36:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:22.642Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"19\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546908263
Jul 17 03:36:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:22.870Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:00-0000:00:01.1-0000:01:00.1-infiniband_mad-umad1.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689546910657
Jul 17 03:36:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:24.056Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx09\", asset_id=\"55\", cpu=\"18\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546914790
Jul 17 03:36:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:24.229Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-tsg-traffic-engine-numa-0-mdqrn\", project=\"TSG-OS\", project_id=\"7\", reason=\"NodeAffinity\", service=\"prometheus-kube-state-metrics\", uid=\"86c61b36-b5a0-422f-be56-b68f56297496\"}" timestamp=1689546911122
Jul 17 03:36:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:24.375Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"mrzcpd_hugepages_setup.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"service\"}" timestamp=1689546911102
Jul 17 03:36:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:24.419Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_created\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-cm-policy-local-cache-kppzx\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"25dfbfd0-9647-4abe-b4a0-2480fdfb4995\"}" timestamp=1689546920068
Jul 17 03:36:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:24.478Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-journald.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689546911102
Jul 17 03:36:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:24.485Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_errs_total\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"vetha475e977\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546914790
Jul 17 03:36:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:24.568Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"nss-user-lookup.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"target\"}" timestamp=1689546911102
Jul 17 03:36:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:24.643Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_processes\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546923200
Jul 17 03:36:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:24.670Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_status_succeeded\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", job_name=\"helm-install-traefik\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546911122
Jul 17 03:36:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:24.741Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"18\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546920160
Jul 17 03:36:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:24.777Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_state_started\", asset=\"msh-tsgx09\", asset_id=\"55\", container=\"helm\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-delete-tsg-traffic-engine-numa-1-bjvt6\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"55e15df4-2d3f-4878-94dc-5fef3e71fbdb\"}" timestamp=1689546920068
Jul 17 03:36:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:24.804Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"prometheus-node-exporter\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Total\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546918611
Jul 17 03:36:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:24.823Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx09\", asset_id=\"55\", container=\"coredns\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Total\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546915241
Jul 17 03:36:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:24.842Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_restarts_total\", asset=\"msh-tsgx09\", asset_id=\"55\", container=\"prometheus-pushgateway\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-pushgateway-bd476bcf9-6ht9n\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"d26afea0-a3cb-45ec-b4d7-06a2ccfaffab\"}" timestamp=1689546920068
Jul 17 03:36:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:24.847Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_max_usage_bytes\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"metrics\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546915253
Jul 17 03:36:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:24.873Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_write_seconds_total\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/shm\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546909208
Jul 17 03:36:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:24.927Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db56\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546917019
Jul 17 03:36:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:24.958Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-bc5fa7cd79f33e11f1d62cbfd0be5ad4c95b759e2dfc75ab24dbdd03f54271ec.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"scope\"}" timestamp=1689546911102
Jul 17 03:36:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:24.989Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"emergency.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"service\"}" timestamp=1689546918416
Jul 17 03:36:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:25.030Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"msh-tsgx09\", asset_id=\"55\", container=\"kube-state-metrics\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"stopped\"}" timestamp=1689546914061
Jul 17 03:36:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:25.055Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_heap_sys_bytes\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546917019
Jul 17 03:36:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:25.057Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_errs_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"lo\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546920160
Jul 17 03:36:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:25.081Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_complete\", asset=\"msh-tsgx09\", asset_id=\"55\", condition=\"true\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", job_name=\"helm-install-traefik\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546920068
Jul 17 03:36:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:25.161Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_running\", asset=\"msh-tsgx09\", asset_id=\"55\", container=\"helm\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-configmap-reloader-84s59\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"47ba69e1-fd09-4e39-941a-89000685484e\"}" timestamp=1689546920068
Jul 17 03:36:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:25.230Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"proc-sys-fs-binfmt_misc.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689546911102
Jul 17 03:36:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:25.282Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_ready\", asset=\"msh-tsgx09\", asset_id=\"55\", container=\"prometheus-server\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-server-6ff784d758-cs9wf\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"58676ba8-2676-43fd-b12b-038d471289b5\"}" timestamp=1689546920068
Jul 17 03:36:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:25.454Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx09\", asset_id=\"55\", cpu=\"64\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546914790
Jul 17 03:36:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:25.533Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_frame_total\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens4f0np0\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546914790
Jul 17 03:36:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:25.596Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-7359daf95b9f7c6d25f09557acc2cbb824033e8e0e6e29a0a2d8a05d2da59202-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689546918416
Jul 17 03:36:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:25.628Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-kernel-tracing.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689546918416
Jul 17 03:36:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:25.761Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_start_time\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-prometheus-glllf\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"8262b103-7c5a-40c8-9681-3282084e7b13\"}" timestamp=1689546911122
Jul 17 03:36:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:25.775Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.grpc.v1.cri-sandboxes-b68f6826fbbf6c5ee858974ef0ea742a94d6b4a83f1bf5bfbdaca46b6f6d374a-shm.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689546911102
Jul 17 03:36:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:25.931Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_labels\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"prometheus-server\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546911122
Jul 17 03:36:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:25.962Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"25\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546920160
Jul 17 03:36:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:25.992Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_created\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"mrzcpd-exporter\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546918744
Jul 17 03:36:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:26.058Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_waiting\", asset=\"twa-tsgx09\", asset_id=\"107\", container=\"metrics\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"cm-policy-local-cache-redis-master-69967b5879-74qzp\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"23f406a6-0255-4082-b091-d1fe8552acb1\"}" timestamp=1689546918744
Jul 17 03:36:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:26.184Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Unknown\", pod=\"helm-install-configmap-reloader-hq5hk\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"cd026eda-ed7f-456d-af88-a83999108711\"}" timestamp=1689546918744
Jul 17 03:36:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:26.343Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"libstoragemgmt.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"simple\", type=\"service\"}" timestamp=1689546911102
Jul 17 03:36:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:26.582Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"pcap-tsgx12\", asset_id=\"117\", condition=\"Available\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"prometheus-server\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"unknown\"}" timestamp=1689546911122
Jul 17 03:36:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:26.691Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_reads_total\", asset=\"twa-tsgx09\", asset_id=\"107\", container=\"firewall\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689546925474
Jul 17 03:36:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:26.696Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_up_changes_total\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth4e5af7f5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546915810
Jul 17 03:36:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:26.699Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-virtual-net-veth9642bab9.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689546911102
Jul 17 03:36:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:26.774Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546915810
Jul 17 03:36:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:26.788Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"rshim.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689546913907
Jul 17 03:36:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:26.846Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-infiniband-umad4.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689546913907
Jul 17 03:36:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:26.916Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"twa-tsgx09\", asset_id=\"107\", condition=\"true\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", pod=\"tsg-traffic-engine-numa-0-firewall-d55dcdb59-dgxgd\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"48a4a0ca-42d3-4487-bc89-7615c4f62e66\"}" timestamp=1689546918744
Jul 17 03:36:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:27.165Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-sda3.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689546913907
Jul 17 03:36:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:27.385Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2did-wwn\\\\x2d0x62cea7f08346b9002c25f712ca485ab5\\\\x2dpart5.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689546913907
Jul 17 03:36:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:27.483Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"44\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546915810
Jul 17 03:36:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:27.545Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_inodes_free\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/run\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546924566
Jul 17 03:36:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:27.558Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"12\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546915810
Jul 17 03:36:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:27.561Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_start_time\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-traefik-crd-m2nsh\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"6ee7b6d8-27b2-4118-9cf0-9229465d0b5b\"}" timestamp=1689546915682
Jul 17 03:36:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:27.563Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"86\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546922882
Jul 17 03:36:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:27.606Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_drop_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"eno2\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546922882
Jul 17 03:36:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:27.634Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_threads_max\", asset=\"twa-tsgx09\", asset_id=\"107\", container=\"configmap-reloader\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546916594
Jul 17 03:36:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:27.637Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", phase=\"Pending\", pod=\"cm-policy-local-cache-redis-master-69967b5879-fpcq2\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"78ab1f97-9ecb-47ed-ae81-048055fb4556\"}" timestamp=1689546915682
Jul 17 03:36:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:27.663Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_multicast_packets_received_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_6\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546922882
Jul 17 03:36:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:27.665Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_working_set_bytes\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"metrics\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546917603
Jul 17 03:36:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:27.698Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_ulimits_soft\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"traefik\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", ulimit=\"max_open_files\"}" timestamp=1689546909795
Jul 17 03:36:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:27.720Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"124\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546922882
Jul 17 03:36:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:27.746Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_down_changes_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens7f1np1\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546922882
Jul 17 03:36:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:27.747Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_packets_received_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_4\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546922882
Jul 17 03:36:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:27.864Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_flags\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth476a8450\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546922882
Jul 17 03:36:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:27.870Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"87\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546922882
Jul 17 03:36:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:27.880Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_bytes_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"br_dign_c\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546922882
Jul 17 03:36:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:27.884Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"83\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546922882
Jul 17 03:36:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:27.914Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_multicast_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"eno33np0\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546922882
Jul 17 03:36:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:27.956Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_labels\", asset=\"msh-tsgx10\", asset_id=\"69\", configmap=\"chart-content-traefik\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546915682
Jul 17 03:36:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:27.964Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_transmit_wait_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_5\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546922882
Jul 17 03:36:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:27.965Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_cpu_seconds_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"slice\"}" timestamp=1689546914114
Jul 17 03:36:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:28.023Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"phy_rx_error_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", dev=\"ens7f0np0\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"mrzcpd-exporter\"}" timestamp=1689546913863
Jul 17 03:36:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:28.414Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_net_dev_group\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"eno1\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546922882
Jul 17 03:36:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:28.435Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_protocol_type\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth6d9f498e\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546922882
Jul 17 03:36:29 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:29.154Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_daemonset_status_current_number_scheduled\", asset=\"msh-tsgx10\", asset_id=\"69\", daemonset=\"kube-multus-ds\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546915682
Jul 17 03:36:29 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:29.172Z caller=dedupe.go:112 component=remote level=error remote_name=52db6f url=http://10.10.10.159:10090/nz-agent/prometheus/proxy/api/v1/write msg="non-recoverable error" count=1000 exemplarCount=0 err="server returned HTTP status 400 : out of order sample"
Jul 17 03:36:29 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:29.484Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_files_free\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda2\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", fstype=\"ext4\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/mnt/onie-boot\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546922882
Jul 17 03:36:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:31.084Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_annotations\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", job_name=\"helm-install-traefik\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546924068
Jul 17 03:36:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:31.133Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kafka_network_RequestMetrics_Errors_total\", asset=\"twa-dellr740-dt002\", asset_id=\"39\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-Kafka-twa-dellr740-dt002\", endpoint_id=\"107\", error=\"NONE\", module=\"DC-Kafka\", module_id=\"5\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", request=\"OffsetFetch\"}" timestamp=1689546929470
Jul 17 03:36:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:31.211Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_labels\", asset=\"msh-tsgx05\", asset_id=\"67\", configmap=\"telegraf-tsg-traffic-engine-numa-0\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546924068
Jul 17 03:36:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:31.291Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"46\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546920939
Jul 17 03:36:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:31.387Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db175\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546930141
Jul 17 03:36:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:31.402Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"scrape_series_added\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_instance=\"localhost\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546915836
Jul 17 03:36:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:31.454Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"msh-tsgx05\", asset_id=\"67\", condition=\"false\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-systemd-exporter-b2sjh\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"b6b97d7c-ea60-4421-aa7a-468442e10f32\"}" timestamp=1689546924068
Jul 17 03:36:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:31.468Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"msh-tsgx05\", asset_id=\"67\", container=\"telegraf\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", state=\"sleeping\"}" timestamp=1689546912772
Jul 17 03:36:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:31.483Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", state=\"sleeping\"}" timestamp=1689546909491
Jul 17 03:36:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:31.491Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"50\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546920939
Jul 17 03:36:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:31.512Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_resident_memory_bytes\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"rdma-ndd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689546921015
Jul 17 03:36:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:31.513Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"112\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546919093
Jul 17 03:36:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:31.520Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-cm-policy-local-cache-g54kj\", project=\"TSG-OS\", project_id=\"7\", reason=\"NodeLost\", service=\"prometheus-kube-state-metrics\", uid=\"b7366540-fb42-43e9-ab99-2918757ab561\"}" timestamp=1689546924068
Jul 17 03:36:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:31.533Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-6bb2158f\\\\x2d52d5\\\\x2d4138\\\\x2d9127\\\\x2d38341f63d806-volume\\\\x2dsubpaths-config\\\\x2dvolume-telegraf-1.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689546926907
Jul 17 03:36:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:31.574Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_data_received_bytes_total\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_2\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546920939
Jul 17 03:36:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:31.750Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.grpc.v1.cri-sandboxes-8d39a9517b918329c7f81c70c48473c00048fa57270b260f86a901ffc615c9d2-shm.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689546921015
Jul 17 03:36:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:31.830Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"msh-tsgx05\", asset_id=\"67\", container=\"metrics\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"stopped\"}" timestamp=1689546905641
Jul 17 03:36:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:31.883Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_metadata_generation\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-1-firewall-6f4cc8b6f\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546924068
Jul 17 03:36:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:32.137Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", phase=\"Running\", pod=\"systemd-exporter-7f549d9485-xfl4z\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"e186b759-1542-4733-9263-82c6ae7fed9e\"}" timestamp=1689546924068
Jul 17 03:36:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:32.163Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_frame_total\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"eno2\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546920939
Jul 17 03:36:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:32.237Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"user.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"slice\"}" timestamp=1689546926907
Jul 17 03:36:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:32.392Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_constraint_errors_transmitted_total\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_10\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546920939
Jul 17 03:36:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:32.437Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db127\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546930141
Jul 17 03:36:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:32.471Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_max_fds\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"exporter-systemd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689546921015
Jul 17 03:36:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:32.474Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_compressed_total\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"virtio_dign_c\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546919093
Jul 17 03:36:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:32.511Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-ask-password-wall.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"simple\", type=\"service\"}" timestamp=1689546926907
Jul 17 03:36:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:32.512Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_mtu_bytes\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth6dcefb50\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546920939
Jul 17 03:36:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:32.532Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db132\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546916711
Jul 17 03:36:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:32.534Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_annotations\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-0-firewall-6789f4bc9b\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546919894
Jul 17 03:36:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:32.560Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Send_Tcp_Rst\", app_name=\"sapp4\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", vsysId=\"1\"}" timestamp=1689546930218
Jul 17 03:36:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:32.568Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"process_start_time_seconds\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546929260
Jul 17 03:36:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:32.601Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_memory_VmallocTotal_bytes\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546920605
Jul 17 03:36:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:32.631Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.grpc.v1.cri-sandboxes-a632288c817034ef9afe274ba19e12b639c521505775aedb4b4d36a71848dc5b-shm.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689546926907
Jul 17 03:36:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:32.648Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_tasks_max\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"tsg-os-HAL.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689546926907
Jul 17 03:36:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:32.693Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-virtual-net-br_dign_s.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689546927691
Jul 17 03:36:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:32.700Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-a632288c817034ef9afe274ba19e12b639c521505775aedb4b4d36a71848dc5b-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689546926907
Jul 17 03:36:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:32.749Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_annotations\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"kubernetes\", uid=\"680ade8c-98d4-4e56-81e3-548831307c80\"}" timestamp=1689546919894
Jul 17 03:36:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:32.758Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"swap.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"target\"}" timestamp=1689546926907
Jul 17 03:36:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:32.783Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"55\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546920605
Jul 17 03:36:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:32.923Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Running\", pod=\"helm-install-ipmi-exporter-qg4pr\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"6d488923-85e6-4a2b-a06b-4a00d69292b7\"}" timestamp=1689546919894
Jul 17 03:36:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:33.180Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"104\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546920605
Jul 17 03:36:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:33.197Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"55\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546920605
Jul 17 03:36:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:33.224Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"twa-tsgx05\", asset_id=\"99\", condition=\"unknown\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-mrzcpd-exporter-dmnbs\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"1982d9e9-ffea-4312-9282-d3a102867809\"}" timestamp=1689546919894
Jul 17 03:36:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:33.259Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"prometheus-pushgateway\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Total\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546904248
Jul 17 03:36:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:33.294Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Read\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689546910914
Jul 17 03:36:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:33.325Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"72\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546920791
Jul 17 03:36:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:33.396Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"41\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546920605
Jul 17 03:36:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:33.398Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2dlabel-TGS\\\\x2dOS\\\\x2dUPDATE.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689546926907
Jul 17 03:36:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:33.399Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"41\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546919093
Jul 17 03:36:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:33.399Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_address_assign_type\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth6bfabd57\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546920791
Jul 17 03:36:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:33.414Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db99\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546929260
Jul 17 03:36:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:33.431Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"smartd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"notify\", type=\"service\"}" timestamp=1689546927691
Jul 17 03:36:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:33.466Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_running\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"helm\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-prometheus-s7vqp\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"8b1286e7-fc1f-4e1e-a7a7-215916c4185e\"}" timestamp=1689546919276
Jul 17 03:36:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:33.482Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-ask-password-wall.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"simple\", type=\"service\"}" timestamp=1689546927691
Jul 17 03:36:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:33.564Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"48\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546920791
Jul 17 03:36:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:33.592Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"ext4\", name=\"data-var-lib-kubelet-pods-bdd10b0d\\\\x2dcc33\\\\x2d4a4f\\\\x2d9622\\\\x2d89ebf267e6c3-volume\\\\x2dsubpaths-config\\\\x2dvolume-firewall-1.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689546927691
Jul 17 03:36:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:33.698Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_io_time_weighted_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda3\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546909424
Jul 17 03:36:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:33.706Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"4\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546920791
Jul 17 03:36:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:33.710Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"9\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546920605
Jul 17 03:36:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:33.718Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_commands_failed_calls_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cmd=\"expire\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546916711
Jul 17 03:36:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:33.731Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sshd-keygen@rsa.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689546927691
Jul 17 03:36:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:33.741Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_bytes_total\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"cadvisor\", interface=\"ens4f1np1\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546916872
Jul 17 03:36:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:33.760Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_waiting\", asset=\"twa-tsgx05\", asset_id=\"99\", container=\"coredns\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"coredns-58679555bf-v6dn2\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"8c5c1473-ca58-435f-a331-a241afdff2f9\"}" timestamp=1689546919894
Jul 17 03:36:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:33.771Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"114\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546920605
Jul 17 03:36:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:33.801Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_system_seconds_total\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"dashboard-metrics-scraper\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546919775
Jul 17 03:36:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:33.813Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_carrier_total\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vetha6bdf7f7\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546920605
Jul 17 03:36:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:33.819Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"local-path-provisioner\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", failure_type=\"pgfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"hierarchy\"}" timestamp=1689546914449
Jul 17 03:36:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:33.832Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"plymouth-quit-wait.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"service\"}" timestamp=1689546927691
Jul 17 03:36:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:33.836Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"85\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546920791
Jul 17 03:36:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:33.850Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2did-scsi\\\\x2d362cea7f0add57d002c2c229b337423e6\\\\x2dpart1.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689546922644
Jul 17 03:36:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:34.000Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-sda3.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689546927691
Jul 17 03:36:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:34.232Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"twa-tsgx05\", asset_id=\"99\", condition=\"false\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-configmap-reloader-csz4v\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"f2a551b0-9b53-4e7a-a0d8-da304384770b\"}" timestamp=1689546919894
Jul 17 03:36:36 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:36.692Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kafka_server_socket_server_metrics_request_rate\", asset=\"pcap-dellr740-dt002\", asset_id=\"33\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-Kafka-pcap-dellr740-dt002\", endpoint_id=\"101\", listener=\"SSL\", module=\"DC-Kafka\", module_id=\"5\", networkProcessor=\"8\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546935788
Jul 17 03:36:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:40.401Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"33\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546931647
Jul 17 03:36:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:40.428Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db83\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546933827
Jul 17 03:36:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:40.524Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_device_id\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"virtio_dign_c\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546931647
Jul 17 03:36:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:40.536Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_errs_total\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"eno34np1\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546931647
Jul 17 03:36:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:40.589Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_queue_length\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"br_dign_s\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546931647
Jul 17 03:36:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:40.662Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"114\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546931647
Jul 17 03:36:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:40.681Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db86\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546933827
Jul 17 03:36:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:40.768Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_reservation_limit_bytes\", asset=\"twa-tsgx04\", asset_id=\"97\", container=\"dign-server-web\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546928797
Jul 17 03:36:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:40.944Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"82\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546931647
Jul 17 03:36:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:41.161Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"nfs-server.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689546935764
Jul 17 03:36:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:41.262Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_10_194_1_metric_scanSize_99_9th_percentile\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546939758
Jul 17 03:36:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:41.283Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", state=\"iowaiting\"}" timestamp=1689546922297
Jul 17 03:36:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:41.327Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_10_193_1_metric_majorCompactionInputFileCount_min\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546939758
Jul 17 03:36:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:41.346Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_created\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_endpoint=\"cm-policy-local-cache-redis-headless\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546940034
Jul 17 03:36:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:41.374Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_owner\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", job_name=\"helm-install-mrzcpd-exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", owner_is_controller=\"<none>\", owner_kind=\"<none>\", owner_name=\"<none>\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546940034
Jul 17 03:36:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:41.401Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_scanCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"rtp_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"c35ab1d140c31d8ea561a5a9503555b9\", table=\"index_time_rtp_hos_bucket\", type=\"index\"}" timestamp=1689546939758
Jul 17 03:36:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:41.502Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_storeCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"7f9e876e99553e036bbfa60f301e051b\", table=\"proxy_hos_bucket\", type=\"data\"}" timestamp=1689546939758
Jul 17 03:36:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:41.529Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_minStoreFileAge\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"session_record_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"587346287ff7fe7466c7378edf2e47ec\", table=\"index_filename_session_record_hos_bucket\", type=\"index\"}" timestamp=1689546939758
Jul 17 03:36:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:41.548Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"9\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546931647
Jul 17 03:36:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:41.817Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"tracefs\", name=\"sys-kernel-tracing.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689546935764
Jul 17 03:36:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:41.849Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2did-wwn\\\\x2d0x62cea7f08346c5002c26f9f8d57b1b40\\\\x2dpart4.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689546935764
Jul 17 03:36:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:41.936Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"85\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546931647
Jul 17 03:36:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:42.020Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"promhttp_metric_handler_requests_total\", asset=\"twa-tsgx04\", asset_id=\"97\", code=\"200\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"ipmi-exporter\"}" timestamp=1689546932488
Jul 17 03:36:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:42.051Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"28\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546931647
Jul 17 03:36:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:42.067Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_filteredReadRequestCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"assessment_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"27576aeb33671b88e7a5aa4de5015e12\", table=\"index_filename_assessment_hos_bucket\", type=\"index\"}" timestamp=1689546939758
Jul 17 03:36:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:42.152Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_LOG_metric_getTime_98th_percentile\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546939758
Jul 17 03:36:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:42.220Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_config_maxmemory\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546933827
Jul 17 03:36:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:42.364Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_10_193_1_metric_flushMemstoreSize_min\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546939758
Jul 17 03:36:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:42.498Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_complete\", asset=\"pcap-tsgx01\", asset_id=\"71\", condition=\"true\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", job_name=\"helm-install-systemd-exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546935154
Jul 17 03:36:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:42.583Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"6\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546936242
Jul 17 03:36:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:42.594Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_10_194_1_metric_minStoreFileAge\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546939758
Jul 17 03:36:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:42.610Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda5\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", fstype=\"ext4\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/data\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546936242
Jul 17 03:36:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:42.617Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", container=\"dign-server-dns\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Read\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546925699
Jul 17 03:36:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:42.617Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", cpu=\"76\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546936242
Jul 17 03:36:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:42.630Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_speed_bytes\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens1f0np0\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546936242
Jul 17 03:36:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:42.639Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", container=\"exporter\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Read\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546936832
Jul 17 03:36:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:42.646Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", cpu=\"43\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546936242
Jul 17 03:36:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:42.651Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failcnt\", asset=\"pcap-tsgx01\", asset_id=\"71\", container=\"redis\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546934322
Jul 17 03:36:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:42.657Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_working_set_bytes\", asset=\"pcap-tsgx01\", asset_id=\"71\", container=\"dign-server-web\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546940442
Jul 17 03:36:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:42.662Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"auth-rpcgss-module.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689546940671
Jul 17 03:36:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:42.673Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_processes\", asset=\"pcap-tsgx01\", asset_id=\"71\", container=\"exporter-merger\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546925707
Jul 17 03:36:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:42.681Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_physical_state_id\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"mlx5_7\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546936242
Jul 17 03:36:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:42.690Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_frame_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"lo\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546936242
Jul 17 03:36:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:42.706Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"plymouth-start.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"forking\", type=\"service\"}" timestamp=1689546940671
Jul 17 03:36:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:42.740Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_rate_bytes_per_second\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"mlx5_0\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546936242
Jul 17 03:36:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:42.786Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_scaling_frequency_hertz\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", cpu=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Gohangout-msh-dellr740-dt002\", endpoint_id=\"167\", module=\"DC-Gohangout\", module_id=\"35\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546942623
Jul 17 03:36:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:43.274Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", container=\"metrics-server\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"cadvisor\", failure_type=\"pgfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"hierarchy\"}" timestamp=1689546940134
Jul 17 03:36:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:43.290Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"pcap-tsgx01\", asset_id=\"71\", condition=\"unknown\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-traefik-6b6lg\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"5f8aea5a-3386-417f-afa8-3bbb60c77409\"}" timestamp=1689546935154
Jul 17 03:36:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:43.291Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_packets_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"cadvisor\", interface=\"cnibr0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546940554
Jul 17 03:36:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:43.305Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_symbol_error_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"mlx5_0\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546936242
Jul 17 03:36:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:43.421Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_virtual_memory_max_bytes\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"udisks2.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689546940671
Jul 17 03:36:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:43.438Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"initrd-switch-root.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689546940671
Jul 17 03:36:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:43.791Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", cpu=\"16\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Gohangout-msh-dellr740-dt002\", endpoint_id=\"167\", mode=\"system\", module=\"DC-Gohangout\", module_id=\"35\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546942623
Jul 17 03:36:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:43.809Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_id\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"br-447915b42434\", endpoint=\"NC-Gohangout-msh-dellr740-dt002\", endpoint_id=\"167\", module=\"DC-Gohangout\", module_id=\"35\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546942623
Jul 17 03:36:47 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:47.549Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"clickhouse_local_thread\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Clickhouse-msh-dellr740-dt001\", endpoint_id=\"79\", module=\"NC-Clickhouse\", module_id=\"29\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546947154
Jul 17 03:36:48 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:48.587Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_session_record_hos_bucket_metric_flushMemstoreSize_SizeRangeCount_100_1000\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546947438
Jul 17 03:36:48 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:48.613Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_193_1_metric_readRequestCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546947438
Jul 17 03:36:48 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:48.686Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_default_metric_scanTime_99_9th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546947438
Jul 17 03:36:48 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:48.703Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_193_1_metric_compactionInputSize_min\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546947438
Jul 17 03:36:48 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:48.846Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_1_metric_appendTime_99th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546947438
Jul 17 03:36:48 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:48.865Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_incrementCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"knowledge_base_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"3dbb80d4d8caba842e6b492ade9b3889\", table=\"index_filename_knowledge_base_hos_bucket\", type=\"index\"}" timestamp=1689546947438
Jul 17 03:36:48 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:48.880Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"java_lang_Threading_ThreadCpuTimeEnabled\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546947438
Jul 17 03:36:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:49.045Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_compactionsCompletedCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"troubleshooting_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"8d05b49601aeb317102c899158fd3ce4\", table=\"index_partfile_troubleshooting_hos_bucket\", type=\"index\"}" timestamp=1689546947438
Jul 17 03:36:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:49.091Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_4_metric_majorCompactionOutputSize_median\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546947438
Jul 17 03:36:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:49.404Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Append_90th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Server\"}" timestamp=1689546947438
Jul 17 03:36:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:49.626Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"121\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546941797
Jul 17 03:36:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:49.650Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx10\", asset_id=\"109\", cpu=\"38\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546941797
Jul 17 03:36:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:49.668Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"promhttp_metric_handler_requests_total\", asset=\"twa-tsgx10\", asset_id=\"109\", code=\"503\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546941797
Jul 17 03:36:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:49.829Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx10\", asset_id=\"109\", cpu=\"93\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546941797
Jul 17 03:36:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:49.901Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db61\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546947556
Jul 17 03:36:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:49.905Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_errs_total\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth6ffe5201\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546941797
Jul 17 03:36:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:49.931Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-virtual-net-vethb8cbe44b.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689546941057
Jul 17 03:36:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:49.942Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dbus.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689546946554
Jul 17 03:36:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:49.964Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-a379a67869d57aedcb0526bd2cb5e1a6fe7faed6994dcda2a2fc2426d6838604-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689546946554
Jul 17 03:36:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:49.969Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", cpu=\"0\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-olap003\", endpoint_id=\"235\", mode=\"user\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"yarn_resourcemanager,zookeeper,nacos,kafka,spark,druid_data,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546949714
Jul 17 03:36:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:50.006Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx10\", asset_id=\"109\", cpu=\"11\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546941797
Jul 17 03:36:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:50.010Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_info\", asset=\"twa-tsgx08\", asset_id=\"105\", created_by_kind=\"ReplicaSet\", created_by_name=\"tsg-traffic-engine-numa-0-firewall-8c9c7fc95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", host_ip=\"10.10.30.168\", host_network=\"false\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", node=\"localhost\", nz_agent_id=\"3\", pod=\"tsg-traffic-engine-numa-0-firewall-8c9c7fc95-6rm9z\", pod_ip=\"10.42.0.122\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"207ceb95-e1ef-4b7e-a587-0a6495b0f134\"}" timestamp=1689546941568
Jul 17 03:36:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:50.018Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_writes_merged_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/var/lib/kubelet/pods/a0e438ae-62dd-4767-974d-b0821c7c31f3/volumes/kubernetes.io~projected/kube-api-access-tlgrw\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546944595
Jul 17 03:36:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:50.023Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx10\", asset_id=\"109\", cpu=\"2\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546941797
Jul 17 03:36:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:50.048Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:20-0000:20:03.1-0000:21:00.0-infiniband-mlx5_2.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689546941057
Jul 17 03:36:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:50.090Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_scanCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"default\", project=\"OLAP\", project_id=\"3\", region=\"1ec6c491dc19e54814be0f747b7ceb69\", table=\"index_time_default\", type=\"index\"}" timestamp=1689546947438
Jul 17 03:36:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:50.091Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"1\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546946709
Jul 17 03:36:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:50.103Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_filteredReadRequestCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"3c853312538c362cb3ee2d2384774162\", table=\"knowledge_base_hos_bucket\", type=\"data\"}" timestamp=1689546947438
Jul 17 03:36:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:50.119Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_1_metric_compactionOutputSize_max\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546947438
Jul 17 03:36:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:50.128Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"overlay\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-d1ba2b94f3b82a618608b67ddf593f4dd2c85c9a409c96e93b300811e565e176-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689546946554
Jul 17 03:36:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:50.147Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2did-scsi\\\\x2d362cea7f08346c1002c270339b2b39591\\\\x2dpart5.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689546946554
Jul 17 03:36:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:50.161Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_reads_total\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"local-path-provisioner\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546925611
Jul 17 03:36:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:50.176Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_minStoreFileAge\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"session_record_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"88a9bbe2136852f7ad790e6c839f28d2\", table=\"index_time_session_record_hos_bucket\", type=\"index\"}" timestamp=1689546947438
Jul 17 03:36:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:50.177Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_minStoreFileAge\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"proxy_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"5668a83f5924df13519f29d7402718b1\", table=\"index_time_proxy_hos_bucket\", type=\"index\"}" timestamp=1689546947438
Jul 17 03:36:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:50.228Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_drop_total\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethe417278f\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546941797
Jul 17 03:36:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:50.250Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_cpu_seconds_total\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"slice\"}" timestamp=1689546941057
Jul 17 03:36:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:50.294Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_carrier_total\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens4f3np3\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546941797
Jul 17 03:36:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:50.327Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_scrape_collector_success\", asset=\"twa-tsgx10\", asset_id=\"109\", collector=\"diskstats\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546941797
Jul 17 03:36:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:50.342Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_write_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/6e4f2f4687bae3bbca1a2a2cbbc5a63313dfd06a46f8ecb30e6b1173c86a1f59/shm\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546944595
Jul 17 03:36:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:50.446Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_labels\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", pod=\"tsg-traffic-engine-numa-1-firewall-58f9975657-n9n7b\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"ba106c44-d5c3-446a-b10b-530b2492f73f\"}" timestamp=1689546939668
Jul 17 03:36:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:50.461Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"7\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546946709
Jul 17 03:36:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:50.473Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_info\", address=\"bc:97:e1:c6:a5:e2\", asset=\"twa-tsgx08\", asset_id=\"105\", broadcast=\"ff:ff:ff:ff:ff:ff\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"eno35np2\", duplex=\"unknown\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operstate=\"down\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546946709
Jul 17 03:36:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:50.476Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_state_started\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"helm\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-mrzcpd-exporter-wp5lj\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"4e8368e7-61a2-4c24-941b-913c52f69e9e\"}" timestamp=1689546939668
Jul 17 03:36:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:50.510Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_created\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.tsg-traffic-engine-numa-1.v1\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546939668
Jul 17 03:36:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:50.560Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-fd4b4291\\\\x2d55db\\\\x2d4894\\\\x2d9795\\\\x2d6abc7f3024db-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2dgtld6.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689546946554
Jul 17 03:36:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:50.600Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"twa-tsgx10\", asset_id=\"109\", cpu=\"88\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546941797
Jul 17 03:36:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:50.626Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"29\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546946709
Jul 17 03:36:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:50.640Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_netstat_Icmp6_InErrors\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546946709
Jul 17 03:36:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:50.645Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2did-wwn\\\\x2d0x62cea7f08346c1002c270339b2b39591.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689546946554
Jul 17 03:36:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:50.660Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"36\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546946709
Jul 17 03:36:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:50.661Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.grpc.v1.cri-sandboxes-1d0a9283baaeb68215db2c28cfe83a4ec2014931320cc1ec38d111b6cd89d7b3-shm.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689546946554
Jul 17 03:36:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:50.744Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_load_average_10s\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689546932938
Jul 17 03:36:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:50.788Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_labels\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_endpoint=\"cm-policy-local-cache-redis-master\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546939668
Jul 17 03:36:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:50.789Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens6f0np0\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546946709
Jul 17 03:36:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:50.805Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_frame_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"virtio_dign_c\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546946709
Jul 17 03:36:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:50.845Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_resource_requests\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"kube-multus\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", node=\"localhost\", nz_agent_id=\"3\", pod=\"kube-multus-ds-p9mt7\", project=\"TSG-OS\", project_id=\"7\", resource=\"memory\", service=\"prometheus-kube-state-metrics\", uid=\"f930767b-550c-482d-9c10-4ec670104c56\", unit=\"byte\"}" timestamp=1689546939668
Jul 17 03:36:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:51.153Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_restarts_total\", asset=\"twa-tsgx08\", asset_id=\"105\", container=\"helm\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-delete-tsg-traffic-engine-numa-1-7nr86\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"de8cc098-4a75-498b-9682-77e8086dba81\"}" timestamp=1689546941568
Jul 17 03:36:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:51.195Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"70\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546946709
Jul 17 03:36:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:51.650Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546946709
Jul 17 03:36:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:53.069Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"msh-tsgx08\", asset_id=\"65\", cpu=\"50\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546948328
Jul 17 03:36:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:53.087Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-6daf80e30594499692da7569f2728f911694f6d03942072ca62498fac314813e.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"scope\"}" timestamp=1689546952135
Jul 17 03:36:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:53.232Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:e0-0000:e0:05.1-0000:e1:00.1-net-eno2.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689546952135
Jul 17 03:36:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:53.275Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_address_assign_type\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"vethae353140\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546948328
Jul 17 03:36:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:53.291Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2did-scsi\\\\x2d362cea7f08346bd002c2c2c420d293f39\\\\x2dpart3.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689546952135
Jul 17 03:36:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:53.410Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", cpu=\"32\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546949337
Jul 17 03:36:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:53.422Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", container=\"kube-multus\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"stopped\"}" timestamp=1689546937732
Jul 17 03:36:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:53.467Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_ports\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_endpoint=\"kube-dns\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", port_name=\"dns-tcp\", port_number=\"53\", port_protocol=\"TCP\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546944478
Jul 17 03:36:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:53.533Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_max_usage_bytes\", asset=\"pcap-tsgx11\", asset_id=\"115\", container=\"dign-server-dns\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546946605
Jul 17 03:36:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:53.541Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"msh-tsgx08\", asset_id=\"65\", container=\"prometheus-alertmanager-configmap-reload\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"running\"}" timestamp=1689546916035
Jul 17 03:36:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:53.570Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-vconsole-setup.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689546951179
Jul 17 03:36:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:53.571Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_packets_transmitted_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"mlx5_6\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546949337
Jul 17 03:36:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:53.588Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-f0fffa29b88fa116bdea37d764c5e83dde02eae3c2feefc11403ebbe72bd24b7.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"scope\"}" timestamp=1689546951179
Jul 17 03:36:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:53.605Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"msh-tsgx12\", asset_id=\"113\", condition=\"unknown\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"dign-server-ml4qh\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"2ed77d82-be58-47ef-9e9a-55c3cc7c3703\"}" timestamp=1689546948663
Jul 17 03:36:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:53.605Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", container=\"configmap-reloader\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Read\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546941636
Jul 17 03:36:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:53.624Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_scrape_collector_success\", asset=\"msh-tsgx12\", asset_id=\"113\", collector=\"nfsd\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546943391
Jul 17 03:36:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:53.661Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx12\", asset_id=\"113\", cpu=\"126\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546943391
Jul 17 03:36:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:53.680Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_net_dev_group\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"vethd3b50248\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546943391
Jul 17 03:36:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:53.692Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"scrape_samples_scraped\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_instance=\"10.42.0.73:9273\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", vsysId=\"1\"}" timestamp=1689546940470
Jul 17 03:36:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:53.730Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dbus.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"simple\", type=\"service\"}" timestamp=1689546951179
Jul 17 03:36:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:53.886Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx08\", asset_id=\"65\", cpu=\"17\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546948328
Jul 17 03:36:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:53.905Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"tmpfs\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/data/var/lib/kubelet/pods/df708fd2-fb21-4af8-958e-4ba1dc3e2b4a/volumes/kubernetes.io~projected/kube-api-access-ct7xl\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546948328
Jul 17 03:36:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:53.921Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"msh-tsgx12\", asset_id=\"113\", container=\"lb-tcp-443\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"iowaiting\"}" timestamp=1689546937031
Jul 17 03:36:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:53.969Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_info\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", version=\"go1.17.3\"}" timestamp=1689546943391
Jul 17 03:36:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:54.005Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_last_seen\", asset=\"msh-tsgx08\", asset_id=\"65\", container=\"mrapm-device\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546937924
Jul 17 03:36:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:54.041Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_local_link_integrity_errors_total\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_1\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546948328
Jul 17 03:36:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:54.060Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_limit_bytes\", asset=\"msh-tsgx08\", asset_id=\"65\", container=\"dign-server-dns\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546937907
Jul 17 03:36:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:54.083Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_multicast_packets_received_total\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_8\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546948328
Jul 17 03:36:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:54.146Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"msh-tsgx08\", asset_id=\"65\", cpu=\"21\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546948328
Jul 17 03:36:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:54.439Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cpu=\"57\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546939409
Jul 17 03:36:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:54.480Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_fifo_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth40581f6b\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546939409
Jul 17 03:36:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:54.522Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_writes_bytes_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", container=\"traefik\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546947938
Jul 17 03:36:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:54.661Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx12\", asset_id=\"113\", cpu=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546943391
Jul 17 03:36:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:54.675Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cpu=\"23\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546939409
Jul 17 03:36:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:54.687Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_address_assign_type\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth4e8b65d2\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546943391
Jul 17 03:36:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:54.702Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_scrape_collector_success\", asset=\"pcap-tsgx07\", asset_id=\"85\", collector=\"os\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546939409
Jul 17 03:36:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:54.708Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"initrd-root-device.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"target\"}" timestamp=1689546940884
Jul 17 03:36:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:55.037Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx12\", asset_id=\"113\", container=\"firewall\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Discard\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689546932581
Jul 17 03:36:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:55.441Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failcnt\", asset=\"pcap-tsgx07\", asset_id=\"85\", container=\"telegraf\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689546934510
Jul 17 03:36:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:55.455Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_up_changes_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"br_dign_c\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546939409
Jul 17 03:36:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:55.458Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_state_started\", asset=\"pcap-tsgx07\", asset_id=\"85\", container=\"telegraf\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", pod=\"tsg-traffic-engine-numa-1-firewall-6c94f9df78-jgzzt\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"bdb49414-5fd6-4eeb-a829-ad76bdcb350b\"}" timestamp=1689546942058
Jul 17 03:36:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:55.473Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cpu=\"98\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546939409
Jul 17 03:36:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:55.495Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cpu=\"34\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546939409
Jul 17 03:36:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:55.543Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_link_error_recovery_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"mlx5_0\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546939409
Jul 17 03:36:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:56.207Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", container=\"mrapm-device\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"uninterruptible\"}" timestamp=1689546944225
Jul 17 03:36:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:56.225Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_annotations\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", job_name=\"helm-install-ipmi-exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546953389
Jul 17 03:36:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:56.537Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_compressed_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens1f1np1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546945883
Jul 17 03:36:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:56.747Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"114\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546945883
Jul 17 03:36:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:56.979Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_owner\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", owner_is_controller=\"true\", owner_kind=\"Deployment\", owner_name=\"prometheus-server\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"prometheus-server-6ff784d758\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546953389
Jul 17 03:36:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:56.988Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-virtual-net-veth6af7c83b.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689546943372
Jul 17 03:36:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:57.047Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_errs_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth958fa935\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546945883
Jul 17 03:36:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:57.092Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"70\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546945883
Jul 17 03:36:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:57.107Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_created\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"kubernetes-dashboard-certs\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546953389
Jul 17 03:36:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:57.203Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"libstoragemgmt.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689546943372
Jul 17 03:36:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:57.340Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"72\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546945883
Jul 17 03:36:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:57.596Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-3050ccdaf8165c965954b1326690caeef6dbd5ccaa63155fbfe7b6a725b3e8bb-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689546943372
Jul 17 03:36:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:57.884Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"113\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546945883
Jul 17 03:36:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:57.907Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_net_dev_group\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"vetha9c0dce8\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546945883
Jul 17 03:36:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:57.925Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", cpu=\"5\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"OLAP-Node-Exporter-pcap-dellr740-dt001\", endpoint_id=\"251\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546957557
Jul 17 03:36:58 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:58.017Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"lldpad.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689546943372
Jul 17 03:36:58 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:58.083Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"59\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546945883
Jul 17 03:36:58 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:58.103Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_packets_transmitted_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"mlx5_7\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546945883
Jul 17 03:36:58 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:58.231Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_file_descriptors\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689546942235
Jul 17 03:36:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:59.250Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_scaling_frequency_max_hertz\", asset=\"msh-dellr740-dt003\", asset_id=\"9\", cpu=\"28\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-dt003\", endpoint_id=\"229\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"hbase_master,clickhouse_data,kafka,gohangout,zookeeper,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546957407
Jul 17 03:36:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:59.376Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_next_gc_bytes\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"OLAP-Node-Exporter-pcap-dellr740-dt001\", endpoint_id=\"251\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546957557
Jul 17 03:36:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:59.674Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"msh-tsgx01\", asset_id=\"51\", container=\"local-path-provisioner\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"cadvisor\", failure_type=\"pgfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"container\"}" timestamp=1689546938173
Jul 17 03:36:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:59.677Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_packets_dropped_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"cadvisor\", interface=\"ens7f1np1\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546952630
Jul 17 03:36:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:59.728Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_errs_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth9b710207\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546949083
Jul 17 03:36:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:59.729Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"msh-tsgx01\", asset_id=\"51\", condition=\"Progressing\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"tsg-traffic-engine-numa-1-firewall\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"false\"}" timestamp=1689546947425
Jul 17 03:36:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:59.748Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_carrier_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens1f0np0\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546949083
Jul 17 03:36:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:59.760Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"12\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546949083
Jul 17 03:36:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:36:59.984Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx01\", asset_id=\"51\", container=\"dashboard-metrics-scraper\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Sync\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546945643
Jul 17 03:37:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:00.120Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"101\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546949083
Jul 17 03:37:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:00.128Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_files\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"shm\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/adf38312f586f9789395221f46cd8b7f6a29e1378158d03549f9e4d136a25c13/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546949083
Jul 17 03:37:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:00.145Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_transmit_wait_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_4\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546949083
Jul 17 03:37:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:00.779Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_memory_Percpu_bytes\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546949083
Jul 17 03:37:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:01.105Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"91\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546949083
Jul 17 03:37:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:01.123Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"15\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546949083
Jul 17 03:37:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:01.138Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_labels\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-alertmanager-6df7f457b6-pffs9\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"e38aa3df-c4d6-4584-badd-68c48f35fb6d\"}" timestamp=1689546947425
Jul 17 03:37:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:01.148Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"115\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546949083
Jul 17 03:37:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:01.187Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"process_open_fds\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546949083
Jul 17 03:37:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:01.232Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_start_time\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-node-exporter-t6jf2\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"21204ec2-df19-472e-9ae1-06104ab28d13\"}" timestamp=1689546947425
Jul 17 03:37:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:01.254Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_mtu_bytes\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"br_dign_c\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546949083
Jul 17 03:37:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:01.277Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-548b900871a45d57101cfb436aec0a3fcaab374253e3af668927e0d3ac20467a.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"scope\"}" timestamp=1689546944879
Jul 17 03:37:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:02.231Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"s3_ttfb_seconds_bucket\", api=\"deleteobject\", asset=\"pcap-kvm-cm001\", asset_id=\"45\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"CM-Minio-pcap-kvm-cm001\", endpoint_id=\"301\", le=\"0.05\", module=\"CM-Minio\", module_id=\"63\", nz_agent_id=\"3\", parent_asset=\"pcap-dellr740-pve001\", parent_asset_id=\"43\", project=\"Central-Management\", project_id=\"5\"}" timestamp=1689546962057
Jul 17 03:37:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:02.377Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx02\", asset_id=\"63\", cpu=\"73\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546958318
Jul 17 03:37:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:02.380Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"msh-tsgx02\", asset_id=\"63\", container=\"redis\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"sleeping\"}" timestamp=1689546942693
Jul 17 03:37:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:02.396Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"62\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546958318
Jul 17 03:37:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:02.399Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_files\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"shm\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/202be1d5820ca6fcad88415c7e5ac2e0f59b2214769b6b6969372b483df27e0e/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546958318
Jul 17 03:37:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:02.641Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_namespace_status_phase\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", phase=\"Terminating\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546951412
Jul 17 03:37:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:03.138Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"net_conntrack_dialer_conn_attempted_total\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", dialer_name=\"321\", instance=\"10.10.20.159:10090\", type=\"global\"}" timestamp=1689546962989
Jul 17 03:37:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:03.252Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_system_seconds_total\", asset=\"msh-tsgx02\", asset_id=\"63\", container=\"kube-multus\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546944625
Jul 17 03:37:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:03.332Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"prometheus_http_request_duration_seconds_bucket\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", handler=\"/api/v1/status/buildinfo\", instance=\"10.10.20.159:10090\", le=\"3\", type=\"global\"}" timestamp=1689546962989
Jul 17 03:37:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:03.406Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_drop_total\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth781c553\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-dt001\", endpoint_id=\"225\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546962988
Jul 17 03:37:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:03.446Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"net_conntrack_dialer_conn_established_total\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", dialer_name=\"369\", instance=\"10.10.20.159:10090\", type=\"global\"}" timestamp=1689546962989
Jul 17 03:37:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:03.457Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_gc_duration_seconds\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-dt001\", endpoint_id=\"225\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\", quantile=\"0\"}" timestamp=1689546962988
Jul 17 03:37:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:03.517Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", cpu=\"18\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-dt001\", endpoint_id=\"225\", mode=\"idle\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546962988
Jul 17 03:37:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:03.810Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"msh-tsgx02\", asset_id=\"63\", condition=\"unknown\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-traefik-ph7rq\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"eacc1d44-839a-43ee-bf5c-0bccf9558d70\"}" timestamp=1689546951412
Jul 17 03:37:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:03.894Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_terminated\", asset=\"msh-tsgx02\", asset_id=\"63\", container=\"redis\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"cm-policy-local-cache-redis-master-69967b5879-h2tzb\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"695dd1c7-6904-4ca3-aa3b-49efdba3b420\"}" timestamp=1689546951412
Jul 17 03:37:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:03.922Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_free_bytes\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"shm\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/520ce657f334485cb6f710335a87b69c84743d6573b58274ee797277ab75c77b/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546958318
Jul 17 03:37:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:03.958Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"78\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546958318
Jul 17 03:37:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:04.138Z caller=dedupe.go:112 component=remote level=info remote_name=52db6f url=http://10.10.10.159:10090/nz-agent/prometheus/proxy/api/v1/write msg="Dropped sample for series that was not explicitly dropped via relabelling" ref=4436650
Jul 17 03:37:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:04.371Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"prometheus_target_scrape_pool_sync_total\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", instance=\"10.10.20.159:10090\", scrape_job=\"asset_ping_115\", type=\"global\"}" timestamp=1689546962989
Jul 17 03:37:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:04.475Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", cpu=\"47\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-dt001\", endpoint_id=\"225\", mode=\"iowait\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546962988
Jul 17 03:37:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:04.494Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", cpu=\"44\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-dt001\", endpoint_id=\"225\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546962988
Jul 17 03:37:05 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:05.738Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"java_lang_MemoryPool_PeakUsage_init\", asset=\"msh-dellr740-olap005\", asset_id=\"19\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Yarn-msh-dellr740-olap005-NodeManager\", endpoint_id=\"373\", module=\"NC-Yarn\", module_id=\"33\", name=\"Code Cache\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546964686
Jul 17 03:37:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:06.704Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-0ef97e877ffa5e49d8eaca8f64114adf3d7fda9f941ba1fa34e3875683d8558e.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"scope\"}" timestamp=1689546953582
Jul 17 03:37:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:06.826Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_packets_total\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth1db7f937\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546963910
Jul 17 03:37:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:06.849Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Ipv6_Bit\", app_name=\"sapp4\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", vsysId=\"1\"}" timestamp=1689546957817
Jul 17 03:37:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:06.906Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"39\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546963910
Jul 17 03:37:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:06.934Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cpu=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546963910
Jul 17 03:37:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:06.986Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_hwmon_temp_celsius\", asset=\"twa-tsgx01\", asset_id=\"91\", chip=\"pci0000:00_0000:00:19_3\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", sensor=\"temp1\", service=\"prometheus-node-exporter\"}" timestamp=1689546963910
Jul 17 03:37:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:07.022Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_timex_pps_calibration_total\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546963910
Jul 17 03:37:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:07.147Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"scrape_series_added\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_instance=\"10.42.0.163:8080\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546961227
Jul 17 03:37:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:07.658Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_sockstat_UDPLITE6_inuse\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546963910
Jul 17 03:37:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:07.682Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-c9cd2d343951639ada75f6442c40ab92c504e7e1a427ac84621b382c47742975-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689546953582
Jul 17 03:37:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:07.714Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_usage_bytes\", asset=\"twa-tsgx01\", asset_id=\"91\", container=\"local-path-provisioner\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546962114
Jul 17 03:37:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:07.754Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_bytes_total\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"cadvisor\", interface=\"ens4f3np3\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546961830
Jul 17 03:37:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:07.786Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-ipmi-exporter-n5l8f\", project=\"TSG-OS\", project_id=\"7\", reason=\"Shutdown\", service=\"prometheus-kube-state-metrics\", uid=\"b0fbdc3c-7bf3-43b8-8574-66286b27a6a0\"}" timestamp=1689546961227
Jul 17 03:37:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:08.086Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_system_seconds_total\", asset=\"twa-tsgx01\", asset_id=\"91\", container=\"monit-stream\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546953048
Jul 17 03:37:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:08.160Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"cortex_blocks_meta_sync_failures_total\", agent_name=\"msh-kvm-nz001\", application=\"cortex\", component=\"querier\", datacenter=\"MSH\", instance=\"127.0.0.1:10093\"}" timestamp=1689546967262
Jul 17 03:37:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:08.280Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db72\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546958290
Jul 17 03:37:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:08.540Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_persistentvolumeclaim_labels\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", persistentvolumeclaim=\"prometheus-alertmanager\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546961227
Jul 17 03:37:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:11.490Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_session_record_hos_bucket_metric_majorCompactionOutputFileCount_min\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546970147
Jul 17 03:37:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:11.546Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_replicaid\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"assessment_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"91055c7f7731c83b1f2dde975c2f174d\", table=\"index_time_assessment_hos_bucket\", type=\"index\"}" timestamp=1689546970147
Jul 17 03:37:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:11.593Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_default_metric_scanTime_median\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546970147
Jul 17 03:37:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:11.619Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_session_record_hos_bucket_metric_getTime_98th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546970147
Jul 17 03:37:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:11.627Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_session_record_hos_bucket_metric_memStoreSize\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546970147
Jul 17 03:37:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:11.664Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_3_metric_compactionOutputFileCount_95th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546970147
Jul 17 03:37:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:11.679Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_FUNCTION_metric_putTime_num_ops\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546970147
Jul 17 03:37:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:11.698Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_numReferenceFiles\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"babc1946f31faf5b4151981143c005cf\", table=\"default\", type=\"data\"}" timestamp=1689546968803
Jul 17 03:37:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:11.803Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_filteredReadRequestCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"rtp_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"68f6a23c6442f4c42783919a62cdbc95\", table=\"index_time_rtp_hos_bucket\", type=\"index\"}" timestamp=1689546970147
Jul 17 03:37:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:11.851Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"java_lang_MemoryPool_CollectionUsage_used\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"CMS Old Gen\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546970147
Jul 17 03:37:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:11.869Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_session_record_hos_bucket_metric_compactedInputBytes\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546970147
Jul 17 03:37:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:12.023Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_194_3_metric_majorCompactionInputSize_max\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546970147
Jul 17 03:37:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:12.338Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_numFilesCompactedCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"proxy_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"2097b60eaa4ffea308d53e366e4242fb\", table=\"index_filename_proxy_hos_bucket\", type=\"index\"}" timestamp=1689546970147
Jul 17 03:37:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:12.347Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_compactionsFailedCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"knowledge_base_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"13a4e6025bf464f6e219ed4362f86f50\", table=\"index_time_knowledge_base_hos_bucket\", type=\"index\"}" timestamp=1689546970147
Jul 17 03:37:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:12.353Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_1_metric_compactionOutputFileCount_75th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546970147
Jul 17 03:37:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:12.364Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_replicaid\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"proxy_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"513c53b10d036c82752eb4ff5811ee84\", table=\"index_partfile_proxy_hos_bucket\", type=\"index\"}" timestamp=1689546970147
Jul 17 03:37:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:12.411Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_FUNCTION_metric_scanTime_99_9th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546970147
Jul 17 03:37:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:12.415Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_maxStoreFileAge\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"firewall_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"7825e373ea56537e53b94051524e36b0\", table=\"index_filename_firewall_hos_bucket\", type=\"index\"}" timestamp=1689546970147
Jul 17 03:37:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:12.448Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_3_metric_compactionOutputFileCount_num_ops\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546970147
Jul 17 03:37:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:12.466Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_session_record_hos_bucket_metric_flushTime_mean\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546970147
Jul 17 03:37:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:12.477Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_4_metric_flushOutputSize_median\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546970147
Jul 17 03:37:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:12.552Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_filename_hostest_192_168_20_194_4_metric_tableWriteQueryPerSecond_5min_rate\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546970147
Jul 17 03:37:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:12.658Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_193_1_metric_majorCompactionOutputFileCount_99th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546970147
Jul 17 03:37:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:12.975Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"twa-dellr740-dt002\", asset_id=\"39\", cpu=\"41\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"OLAP-Node-Exporter-twa-dellr740-dt002\", endpoint_id=\"259\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546972055
Jul 17 03:37:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:15.292Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_max_usage_bytes\", asset=\"twa-tsgx07\", asset_id=\"103\", container=\"telegraf-security\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689546953764
Jul 17 03:37:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:15.324Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_name_assign_type\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth9bf7720f\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546961230
Jul 17 03:37:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:15.405Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"35\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546961230
Jul 17 03:37:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:15.495Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_packets_received_total\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_7\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546961230
Jul 17 03:37:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:15.924Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cryptsetup.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"target\"}" timestamp=1689546967046
Jul 17 03:37:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:16.208Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_running\", asset=\"twa-tsgx07\", asset_id=\"103\", container=\"kubernetes-dashboard\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", pod=\"kubernetes-dashboard-66dd5b456-xp4jt\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"4e581f4d-f643-47e4-804b-dd83bbeb1537\"}" timestamp=1689546968076
Jul 17 03:37:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:16.260Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_observed_generation\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"systemd-exporter\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546968076
Jul 17 03:37:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:16.314Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_down_changes_total\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"eno2\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546961230
Jul 17 03:37:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:16.335Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_user_seconds_total\", asset=\"twa-tsgx07\", asset_id=\"103\", container=\"firewall\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689546969103
Jul 17 03:37:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:16.350Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_reads_bytes_total\", asset=\"twa-tsgx07\", asset_id=\"103\", container=\"firewall\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689546969103
Jul 17 03:37:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:16.351Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"plymouth-start.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"service\"}" timestamp=1689546967046
Jul 17 03:37:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:16.387Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_annotations\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"shaping-announce-port-tsg-traffic-engine-numa-1\", uid=\"e98cfd2f-f10b-4cce-a718-800c5a17d9f3\"}" timestamp=1689546968076
Jul 17 03:37:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:16.404Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_speed_bytes\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethdfe9a615\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546961230
Jul 17 03:37:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:16.422Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-ask-password-plymouth.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689546967046
Jul 17 03:37:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:16.443Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"-.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689546967046
Jul 17 03:37:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:16.491Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_terminated\", asset=\"twa-tsgx07\", asset_id=\"103\", container=\"exporter-merger\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"mrzcpd-exporter-5cf7f967cc-95k2q\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"a324e348-d97e-48ff-833e-cccb478bc4df\"}" timestamp=1689546968076
Jul 17 03:37:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:16.507Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_labels\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-server\", uid=\"a0d51bed-5bc4-4c39-984b-6569248c738f\"}" timestamp=1689546968076
Jul 17 03:37:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:16.581Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_heap_released_bytes\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546966027
Jul 17 03:37:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:19.493Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_protocol_type\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth52173962\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546977612
Jul 17 03:37:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:19.679Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"74\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546977612
Jul 17 03:37:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:19.691Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_virtual_memory_bytes\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"rdma-ndd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689546964356
Jul 17 03:37:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:19.901Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx06\", asset_id=\"101\", container=\"dign-client\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Write\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546968971
Jul 17 03:37:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:19.999Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_heap_released_bytes\", asset=\"msh-kvm-cm001\", asset_id=\"3\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"CM-Verify-Policy-msh-kvm-cm001\", endpoint_id=\"295\", module=\"CM-Verify-Policy\", module_id=\"61\", nz_agent_id=\"3\", parent_asset=\"msh-dellr740-pve001\", parent_asset_id=\"1\", project=\"Central-Management\", project_id=\"5\"}" timestamp=1689546978599
Jul 17 03:37:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:20.089Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-netns-cni\\\\x2dd60cd720\\\\x2dd776\\\\x2d5439\\\\x2d7c7e\\\\x2d3f1e630c3a2c.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689546964356
Jul 17 03:37:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:20.150Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_restart_policy\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-traefik-l5xzw\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", type=\"OnFailure\", uid=\"11a6aee2-f492-41ef-931d-e00b56122be3\"}" timestamp=1689546973949
Jul 17 03:37:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:20.186Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Running\", pod=\"coredns-7bb96695bb-vq4ks\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"19dc02d9-2c27-4920-87eb-18d8c7907be0\"}" timestamp=1689546973949
Jul 17 03:37:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:20.271Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"82\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546977612
Jul 17 03:37:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:20.367Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"100\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546977612
Jul 17 03:37:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:20.547Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"phy_tx_error_total\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", dev=\"ens1f1np1\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"mrzcpd-exporter\"}" timestamp=1689546977986
Jul 17 03:37:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:20.645Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_id\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth3177b6ed\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546977612
Jul 17 03:37:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:20.732Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"tsg_master_del\", app_name=\"tsg_master\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"traffic-engine\", line_name=\"dns_response\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", table_name=\"sync_exdata\", vsysId=\"1\"}" timestamp=1689546979077
Jul 17 03:37:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:20.907Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_protocol_type\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth36380b9e\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546968263
Jul 17 03:37:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:20.925Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sshd-keygen@rsa.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689546970657
Jul 17 03:37:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:21.064Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_swap_limit_bytes\", asset=\"twa-tsgx06\", asset_id=\"101\", container=\"lb-tcp-443\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546972894
Jul 17 03:37:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:21.082Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_restarts_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", container=\"telegraf-security\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", pod=\"tsg-traffic-engine-numa-0-firewall-6d4969c56f-54wcn\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"617272f1-e26a-4d41-b763-554ea7e04473\"}" timestamp=1689546973828
Jul 17 03:37:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:21.097Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_daemonset_status_number_unavailable\", asset=\"pcap-tsgx10\", asset_id=\"89\", daemonset=\"kube-multus-ds\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546973828
Jul 17 03:37:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:21.099Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_replicas\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"prometheus-kube-state-metrics-6b768c7bf8\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546973828
Jul 17 03:37:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:21.135Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"122\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546968263
Jul 17 03:37:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:21.161Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_up_changes_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"vetha31a3f11\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546968263
Jul 17 03:37:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:21.166Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", container=\"coredns\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Read\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546968639
Jul 17 03:37:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:21.180Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_unicast_packets_received_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"mlx5_10\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546968263
Jul 17 03:37:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:21.182Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_compressed_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth0d28b4f8\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546968263
Jul 17 03:37:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:21.198Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_protocol_type\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"eno1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546968263
Jul 17 03:37:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:21.202Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"3\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546968263
Jul 17 03:37:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:21.239Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_files\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"shm\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/4e3805e7e9c834e45b85e0af49582b94825380a5a3cf0937584e7bc70eadbc73/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546968263
Jul 17 03:37:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:21.245Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_created\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.ipmi-exporter.v1\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546973828
Jul 17 03:37:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:21.253Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_daemonset_status_number_unavailable\", asset=\"pcap-tsgx10\", asset_id=\"89\", daemonset=\"prometheus-node-exporter\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546973828
Jul 17 03:37:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:21.265Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"28\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546968263
Jul 17 03:37:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:21.279Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_compressed_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens8f0np0\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546968263
Jul 17 03:37:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:21.300Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", container=\"configmap-reloader\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Sync\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546968614
Jul 17 03:37:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:21.416Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas_available\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"prometheus-pushgateway\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546973828
Jul 17 03:37:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:21.540Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"pcap-tsgx10\", asset_id=\"89\", condition=\"Progressing\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"tsg-traffic-engine-numa-0-firewall\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"true\"}" timestamp=1689546973828
Jul 17 03:37:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:21.540Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_threads\", asset=\"pcap-tsgx10\", asset_id=\"89\", container=\"lb-tcp-80\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546973117
Jul 17 03:37:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:21.559Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_restarts_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", container=\"helm\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-tsg-traffic-engine-numa-0-d2b2s\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"7c729184-3a15-4a27-8e65-311777e49481\"}" timestamp=1689546973828
Jul 17 03:37:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:21.776Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_protocol_type\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"eno2\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546968263
Jul 17 03:37:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:21.790Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_compressed_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"eno33np0\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546968263
Jul 17 03:37:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:21.802Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_receive_switch_relay_errors_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"mlx5_5\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546968263
Jul 17 03:37:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:21.817Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_rejected_connections_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546974840
Jul 17 03:37:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:21.837Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"jvm_memory_used_bytes\", application=\"galaxy-job-executor\", area=\"heap\", asset=\"msh-dellr740-olap002\", asset_id=\"13\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"Galaxy-job-executor-msh-dellr740-olap002\", endpoint_id=\"53\", id=\"PS Eden Space\", module=\"Galaxy-job-executor\", module_id=\"19\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546981271
Jul 17 03:37:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:21.968Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_net_dev_group\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth0aa05a07\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546968263
Jul 17 03:37:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:22.086Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"snmptrapd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689546970657
Jul 17 03:37:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:22.096Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Sync\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689546966456
Jul 17 03:37:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:22.109Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kdump.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689546970657
Jul 17 03:37:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:23.520Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"29\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546974790
Jul 17 03:37:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:24.143Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"20\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546980160
Jul 17 03:37:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:24.174Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-2e438e2d86de3c658a3d680f577f8e1e8e7876c5c1cb1fb878b12b31c449954a.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"scope\"}" timestamp=1689546971102
Jul 17 03:37:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:24.242Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"msh-tsgx09\", asset_id=\"55\", cpu=\"87\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546974790
Jul 17 03:37:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:24.313Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_spec_strategy_rollingupdate_max_surge\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"prometheus-alertmanager\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546971122
Jul 17 03:37:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:24.329Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_gc_sys_bytes\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"ipmi-exporter\"}" timestamp=1689546983120
Jul 17 03:37:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:24.348Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_spec_strategy_rollingupdate_max_surge\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"traefik\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546980068
Jul 17 03:37:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:24.362Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_ulimits_soft\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"dashboard-metrics-scraper\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", ulimit=\"max_open_files\"}" timestamp=1689546980790
Jul 17 03:37:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:24.382Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_fifo_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens1f1np1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546980160
Jul 17 03:37:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:24.407Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db216\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546977019
Jul 17 03:37:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:24.442Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_virtual_memory_bytes\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"packagekit.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689546978416
Jul 17 03:37:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:24.468Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx09\", asset_id=\"55\", container=\"coredns\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Read\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546974549
Jul 17 03:37:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:24.482Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_type\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"chart-values-tsg-traffic-engine-numa-0\", service=\"prometheus-kube-state-metrics\", type=\"Opaque\"}" timestamp=1689546971122
Jul 17 03:37:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:24.523Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:80-0000:80:01.1-0000:81:00.0-host0-target0:2:0-0:2:0:0-block-sda-sda4.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689546978416
Jul 17 03:37:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:24.570Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_processes\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689546977289
Jul 17 03:37:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:24.612Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx09\", asset_id=\"55\", container=\"dign-server-dns\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Read\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546968662
Jul 17 03:37:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:24.775Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_load_average_10s\", asset=\"msh-tsgx09\", asset_id=\"55\", container=\"telegraf-security\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689546966200
Jul 17 03:37:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:24.933Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"msh-tsgx09\", asset_id=\"55\", cpu=\"49\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546974790
Jul 17 03:37:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:24.975Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_scrape_collector_success\", asset=\"msh-tsgx09\", asset_id=\"55\", collector=\"edac\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546974790
Jul 17 03:37:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:25.044Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_init_container_status_waiting\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"tsg-dign-init\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"dign-server-bhs24\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"78d32c4f-349d-455c-932b-b75d141ec6e9\"}" timestamp=1689546971122
Jul 17 03:37:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:25.053Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_status_start_time\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", job_name=\"helm-install-cm-policy-local-cache\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546971122
Jul 17 03:37:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:25.064Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"pcap-tsgx12\", asset_id=\"117\", condition=\"false\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"configmap-reloader-6c55bbc49f-fh258\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"f7af3cd7-b972-4aba-a12a-7b5c827e57eb\"}" timestamp=1689546971122
Jul 17 03:37:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:25.073Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx09\", asset_id=\"55\", cpu=\"23\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546974790
Jul 17 03:37:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:25.104Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_multicast_packets_transmitted_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"mlx5_7\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546980160
Jul 17 03:37:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:25.156Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"tmpfs\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/var/lib/kubelet/pods/8faffc72-4ba1-4272-9289-0077b5e77c0a/volumes/kubernetes.io~projected/kube-api-access-h9dld\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546974790
Jul 17 03:37:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:25.172Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas_updated\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"mrzcpd-exporter\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546971122
Jul 17 03:37:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:25.173Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_device_id\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens4f2np2\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546974790
Jul 17 03:37:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:25.215Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link_mode\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"vethc0962f36\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546980160
Jul 17 03:37:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:25.216Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_avail_bytes\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"tmpfs\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546980160
Jul 17 03:37:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:25.230Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"plymouth-quit.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"service\"}" timestamp=1689546971102
Jul 17 03:37:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:25.266Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"msh-kvm-cm001\", asset_id=\"3\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db152\", endpoint=\"CM-Sync-Redis-msh-kvm-cm001\", endpoint_id=\"307\", host=\"192.168.20.157\", module=\"CM-Sync-Redis\", module_id=\"67\", nz_agent_id=\"3\", parent_asset=\"msh-dellr740-pve001\", parent_asset_id=\"1\", project=\"Central-Management\", project_id=\"5\"}" timestamp=1689546983344
Jul 17 03:37:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:25.271Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_packets_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens8f0np0\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546980160
Jul 17 03:37:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:25.306Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_drop_total\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens1f0np0\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546974790
Jul 17 03:37:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:25.306Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_reservation_limit_bytes\", asset=\"msh-tsgx09\", asset_id=\"55\", container=\"configmap-reloader\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546978586
Jul 17 03:37:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:25.324Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"msh-tsgx09\", asset_id=\"55\", container=\"dign-client\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"sleeping\"}" timestamp=1689546972534
Jul 17 03:37:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:25.397Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_restarts_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"redis\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"cm-policy-local-cache-redis-master-69967b5879-8m6nj\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"ebdf9219-42ed-4a26-8500-14f2dacdb9fe\"}" timestamp=1689546971122
Jul 17 03:37:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:25.643Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-prometheus-glllf\", project=\"TSG-OS\", project_id=\"7\", reason=\"NodeLost\", service=\"prometheus-kube-state-metrics\", uid=\"8262b103-7c5a-40c8-9681-3282084e7b13\"}" timestamp=1689546971122
Jul 17 03:37:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:25.657Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_threads\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"telegraf-security\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689546969408
Jul 17 03:37:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:25.674Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_info\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_endpoint=\"prometheus-kube-state-metrics\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546971122
Jul 17 03:37:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:26.049Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db64\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546983549
Jul 17 03:37:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:26.083Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-c3450e7c3a965996a1b4712dfe43e7b9d34640c4616e39cb5d49acf0b5c8c01a.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"scope\"}" timestamp=1689546973907
Jul 17 03:37:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:26.107Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_reads_bytes_total\", asset=\"twa-tsgx09\", asset_id=\"107\", container=\"firewall\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689546974936
Jul 17 03:37:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:26.147Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_fully_labeled_replicas\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"prometheus-pushgateway-bd476bcf9\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546978744
Jul 17 03:37:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:26.262Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_protocol_type\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens6f1np1\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546975810
Jul 17 03:37:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:26.335Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_labels\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_endpoint=\"shaping-announce-port-tsg-traffic-engine-numa-0\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546978744
Jul 17 03:37:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:26.347Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_created\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", pod=\"dashboard-metrics-scraper-5b94897dc9-2zm7m\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"e2fa51ee-442f-45b7-9aac-868fa0aef812\"}" timestamp=1689546978744
Jul 17 03:37:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:26.364Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_ready_replicas\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-1-firewall-6b6bf4679c\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546978744
Jul 17 03:37:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:26.600Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"69\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546975810
Jul 17 03:37:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:26.628Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"mlocate-updatedb.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"simple\", type=\"service\"}" timestamp=1689546973907
Jul 17 03:37:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:26.648Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_spec_replicas\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-1-firewall-84df67cfdd\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546978744
Jul 17 03:37:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:26.742Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"126\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546975810
Jul 17 03:37:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:26.892Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-sda5.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689546973907
Jul 17 03:37:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:27.524Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_oom_events_total\", asset=\"twa-tsgx09\", asset_id=\"107\", container=\"prometheus-alertmanager\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546985127
Jul 17 03:37:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:27.525Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_ulimits_soft\", asset=\"twa-tsgx09\", asset_id=\"107\", container=\"metrics-server\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", ulimit=\"max_open_files\"}" timestamp=1689546984596
Jul 17 03:37:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:27.613Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"35\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546975810
Jul 17 03:37:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:27.634Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_commands_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cmd=\"info\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546983549
Jul 17 03:37:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:27.831Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_node_status_condition\", asset=\"msh-tsgx10\", asset_id=\"69\", condition=\"PIDPressure\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", node=\"localhost\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"true\"}" timestamp=1689546975682
Jul 17 03:37:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:27.847Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_ready\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"redis\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"cm-policy-local-cache-redis-master-69967b5879-fpcq2\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"78ab1f97-9ecb-47ed-ae81-048055fb4556\"}" timestamp=1689546975682
Jul 17 03:37:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:27.887Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"abrtd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"service\"}" timestamp=1689546974114
Jul 17 03:37:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:27.914Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_writes_bytes_total\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"firewall\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689546970525
Jul 17 03:37:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:27.947Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_up\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth9db5012c\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546982882
Jul 17 03:37:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:27.951Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_cpu_shares\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"prometheus-pushgateway\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546984841
Jul 17 03:37:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:27.989Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_size_bytes\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"shm\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/d0eaf79bd8f0fdf3354476645a85ca968217972ef9030249453a97f71ad68165/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546982882
Jul 17 03:37:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:28.025Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_tasks_max\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"lvm2-monitor.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689546974114
Jul 17 03:37:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:28.026Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_metadata_generation\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"systemd-exporter-7f549d9485\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546975682
Jul 17 03:37:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:28.255Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_threads\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"prometheus-alertmanager-configmap-reload\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546973811
Jul 17 03:37:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:28.358Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_running\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"exporter-merger\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"mrzcpd-exporter-5cf7f967cc-674zn\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"05a8d67c-e611-49c0-83d9-d8b010e1bac4\"}" timestamp=1689546975682
Jul 17 03:37:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:28.377Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"mlocate-updatedb.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"service\"}" timestamp=1689546974114
Jul 17 03:37:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:28.415Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_flags\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth481e9e69\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546982882
Jul 17 03:37:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:28.450Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_labels\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"ipmi-exporter\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546975682
Jul 17 03:37:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:28.552Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:00-0000:00:01.1-0000:01:00.1-infiniband_mad-umad1.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689546974114
Jul 17 03:37:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:28.583Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_writes_merged_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546974824
Jul 17 03:37:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:28.627Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_labels\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_endpoint=\"systemd-exporter\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546975682
Jul 17 03:37:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:28.714Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546982882
Jul 17 03:37:29 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:29.568Z caller=dedupe.go:112 component=remote level=error remote_name=52db6f url=http://10.10.10.159:10090/nz-agent/prometheus/proxy/api/v1/write msg="non-recoverable error" count=1000 exemplarCount=0 err="server returned HTTP status 400 : out of order sample"
Jul 17 03:37:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:31.301Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_scrape_collector_success\", asset=\"msh-tsgx05\", asset_id=\"67\", collector=\"uname\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546980939
Jul 17 03:37:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:31.520Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_swap\", asset=\"msh-tsgx05\", asset_id=\"67\", container=\"kubernetes-dashboard\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546972988
Jul 17 03:37:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:31.576Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db191\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546990141
Jul 17 03:37:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:31.652Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-modules-load.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689546986907
Jul 17 03:37:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:31.670Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"62\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546979093
Jul 17 03:37:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:31.703Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"tmpfs\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/var/lib/kubelet/pods/acb0a639-453c-4a2c-a1c6-f36ebe18181d/volumes/kubernetes.io~projected/kube-api-access-xw2dg\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546980939
Jul 17 03:37:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:31.725Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"nfs-idmapd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"forking\", type=\"service\"}" timestamp=1689546986907
Jul 17 03:37:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:31.853Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_errs_total\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens2f1np1\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546980791
Jul 17 03:37:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:31.876Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-6d138495\\\\x2d2ef4\\\\x2d43b8\\\\x2daa40\\\\x2d820b8611ffff-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2dxqbgd.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689546981015
Jul 17 03:37:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:32.053Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_info\", asset=\"twa-tsgx03\", asset_id=\"95\", cluster_ip=\"10.43.243.102\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"metrics-server\", uid=\"fdda4ae8-71bc-4a66-ac27-c4774d0b11fb\"}" timestamp=1689546990436
Jul 17 03:37:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:32.054Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"24\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546979093
Jul 17 03:37:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:32.104Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"vdo.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"oneshot\", type=\"service\"}" timestamp=1689546981015
Jul 17 03:37:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:32.188Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_colls_total\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"br_dign_c\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546980791
Jul 17 03:37:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:32.194Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"85\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546980939
Jul 17 03:37:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:32.273Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"66\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546979093
Jul 17 03:37:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:32.324Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_memory_Active_anon_bytes\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546980939
Jul 17 03:37:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:32.356Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"k3s.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"notify\", type=\"service\"}" timestamp=1689546982644
Jul 17 03:37:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:32.356Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_bytes_total\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", interface=\"ens4f3np3\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546978144
Jul 17 03:37:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:32.434Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_device_id\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth9a152f90\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546980791
Jul 17 03:37:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:32.450Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"telegraf-security\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", state=\"iowaiting\"}" timestamp=1689546965262
Jul 17 03:37:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:32.594Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"34\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546980939
Jul 17 03:37:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:32.596Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"94\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546980939
Jul 17 03:37:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:32.609Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"70\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546980939
Jul 17 03:37:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:32.615Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kdump.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"service\"}" timestamp=1689546982644
Jul 17 03:37:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:32.639Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-podb7e44624_9b20_4189_9ac3_b8a15c1776a6.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"slice\"}" timestamp=1689546982644
Jul 17 03:37:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:32.641Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"twa-tsgx03\", asset_id=\"95\", container=\"lb-tcp-443\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"uninterruptible\"}" timestamp=1689546964152
Jul 17 03:37:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:32.663Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"87\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546980605
Jul 17 03:37:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:32.741Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"twa-tsgx05\", asset_id=\"99\", condition=\"false\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-prometheus-jgzhf\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"f2f577b5-5389-48c3-9e35-cfcac2fb3148\"}" timestamp=1689546979894
Jul 17 03:37:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:32.900Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_size_bytes\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"shm\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/ceca645e6975d39f0681d6e1a0265568d8df6702f42db7fe01963ad704c12709/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546980605
Jul 17 03:37:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:32.904Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_restarts_total\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"prometheus-pushgateway\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-pushgateway-bd476bcf9-4tmc9\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"345ff5d0-20c3-4ba4-bfcb-02d0c4070a47\"}" timestamp=1689546979276
Jul 17 03:37:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:32.925Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_persistentvolume_created\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", persistentvolume=\"pvc-0f67f9a1-1515-49ba-bfe0-9e91f648a16f\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546990436
Jul 17 03:37:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:32.970Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_init_container_info\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"firewall-init\", container_id=\"containerd://5b15c445e9916307f369a72efa14821f3666968adeaf800129f6599fd798375a\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", image=\"registry.gdnt-cloud.website/tsg-init:v23.07-rc1-079e964\", image_id=\"sha256:c3a7da59ebe9d7390fc09b3b4d50c14674963d0d9616b631fe8029ed3bd0e7d1\", image_spec=\"registry.gdnt-cloud.website/tsg-init:v23.07-rc1-079e964\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", pod=\"tsg-traffic-engine-numa-0-firewall-99cddfc69-25wpt\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"abf27d03-a067-448d-81a8-11df769a2208\"}" timestamp=1689546979276
Jul 17 03:37:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:33.023Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"96\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546979093
Jul 17 03:37:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:33.024Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_fifo_total\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"cnibr0\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546980791
Jul 17 03:37:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:33.040Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-infiniband-umad5.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689546986907
Jul 17 03:37:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:33.054Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sysstat.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"service\"}" timestamp=1689546981015
Jul 17 03:37:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:33.058Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_name_assign_type\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth2e01d29e\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546980939
Jul 17 03:37:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:33.072Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_virtual_memory_max_bytes\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-journald.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689546981015
Jul 17 03:37:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:33.092Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"tsg_master_drop_log_s\", app_name=\"tsg_master\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"traffic-engine\", line_name=\"SESSION-RECORD\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", table_name=\"sendlog\", vsysId=\"1\"}" timestamp=1689546989611
Jul 17 03:37:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:33.200Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_name_assign_type\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"eno36np3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546980939
Jul 17 03:37:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:33.799Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Read\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689546980677
Jul 17 03:37:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:33.858Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546979093
Jul 17 03:37:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:33.909Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_user_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", container=\"firewall\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689546972515
Jul 17 03:37:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:33.945Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_changes_total\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth01f00b07\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546980791
Jul 17 03:37:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:33.969Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"126\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546980791
Jul 17 03:37:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:33.973Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", phase=\"Unknown\", pod=\"configmap-reloader-6c55bbc49f-qmcwt\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"03b47e9e-b9a5-462d-b6ff-6e3865bd5555\"}" timestamp=1689546990436
Jul 17 03:37:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:34.000Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"75\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546979093
Jul 17 03:37:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:34.010Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"twa-tsgx03\", asset_id=\"95\", container=\"kube-multus\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"uninterruptible\"}" timestamp=1689546963078
Jul 17 03:37:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:34.054Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_labels\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-server\", uid=\"4c68b520-6501-44fd-8cda-8537b19effa8\"}" timestamp=1689546979276
Jul 17 03:37:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:34.143Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_waiting\", asset=\"twa-tsgx05\", asset_id=\"99\", container=\"helm\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-traefik-crd-bf9k4\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"39ad39af-b9c7-4eb1-a7db-349fcd19b742\"}" timestamp=1689546979894
Jul 17 03:37:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:34.172Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_excessive_buffer_overrun_errors_total\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"mlx5_8\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546980791
Jul 17 03:37:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:34.199Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"124\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546980605
Jul 17 03:37:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:34.419Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db176\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546976711
Jul 17 03:37:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:34.499Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_disk_writes_merged_total\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"sda\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546980605
Jul 17 03:37:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:34.531Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"rpcbind.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"target\"}" timestamp=1689546987691
Jul 17 03:37:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:34.840Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_sector_writes_total\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546978144
Jul 17 03:37:35 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:35.294Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"82\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546980605
Jul 17 03:37:35 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:35.625Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"twa-tsgx05\", asset_id=\"99\", container=\"lb-tcp-443\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"sleeping\"}" timestamp=1689546961963
Jul 17 03:37:39 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:39.353Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_commands_total\", asset=\"msh-kvm-cm001\", asset_id=\"3\", cmd=\"exec\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"CM-Branch-Redis-msh-kvm-cm001\", endpoint_id=\"319\", host=\"192.168.20.157\", module=\"CM-Branch-Redis\", module_id=\"73\", nz_agent_id=\"3\", parent_asset=\"msh-dellr740-pve001\", parent_asset_id=\"1\", project=\"Central-Management\", project_id=\"5\"}" timestamp=1689546998579
Jul 17 03:37:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:40.468Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"77\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546991647
Jul 17 03:37:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:41.791Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_TotalCallTime_98th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-Hmaster\", endpoint_id=\"339\", module=\"DC-HBase\", module_id=\"3\", name=\"Master\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"IPC\"}" timestamp=1689547000906
Jul 17 03:37:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:41.808Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_hbase_table_rsgroup_metric_compactionInputFileCount_min\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546999757
Jul 17 03:37:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:41.905Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_maxCompactionQueueSize\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"assessment_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"dd5914aa905ba6b1b33ca95f2ff471c2\", table=\"index_filename_assessment_hos_bucket\", type=\"index\"}" timestamp=1689546999757
Jul 17 03:37:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:41.921Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_LoginFailureAvgTime\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"UgiMetrics\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689546999757
Jul 17 03:37:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:41.957Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_avgStoreFileAge\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"firewall_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"54827284b07aa8de7179e562053df579\", table=\"index_time_firewall_hos_bucket\", type=\"index\"}" timestamp=1689546999757
Jul 17 03:37:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:41.985Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_10_194_1_metric_incrementTime_25th_percentile\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546999757
Jul 17 03:37:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:42.091Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_10_194_1_metric_majorCompactionInputSize_99th_percentile\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546999757
Jul 17 03:37:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:42.091Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_info\", address=\"bc:97:e1:0c:06:90\", asset=\"twa-tsgx04\", asset_id=\"97\", broadcast=\"ff:ff:ff:ff:ff:ff\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens4f0np0\", duplex=\"unknown\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operstate=\"down\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546991647
Jul 17 03:37:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:42.197Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_compactionsCompletedCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"default\", project=\"OLAP\", project_id=\"3\", region=\"e88ff594a6777bae70780cedc37452c0\", table=\"index_time_default\", type=\"index\"}" timestamp=1689546999757
Jul 17 03:37:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:42.200Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"twa-tsgx04\", asset_id=\"97\", condition=\"Available\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"coredns\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"true\"}" timestamp=1689547000034
Jul 17 03:37:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:42.223Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_system_table_bucket_meta_metric_compactionInputFileCount_75th_percentile\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546999757
Jul 17 03:37:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:42.294Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_file_descriptors\", asset=\"twa-tsgx04\", asset_id=\"97\", container=\"dign-server-web\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546976813
Jul 17 03:37:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:42.313Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_ports\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_endpoint=\"metrics-server\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", port_name=\"https\", port_number=\"10250\", port_protocol=\"TCP\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689547000034
Jul 17 03:37:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:42.429Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"118\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689546996242
Jul 17 03:37:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:42.450Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_multicast_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens6f0np0\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546996242
Jul 17 03:37:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:42.551Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_spec_replicas\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"mrzcpd-exporter\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546995154
Jul 17 03:37:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:42.571Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", cpu=\"29\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546996242
Jul 17 03:37:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:42.600Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-sda5.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689546995764
Jul 17 03:37:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:42.604Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_speed_bytes\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth2f178825\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546991647
Jul 17 03:37:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:42.641Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_node_info\", asset=\"twa-tsgx04\", asset_id=\"97\", container_runtime_version=\"containerd://1.7.1-k3s1\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", internal_ip=\"10.10.30.164\", kernel_version=\"5.17.15-1.el8.x86_64\", kubelet_version=\"v1.27.2+k3s1\", kubeproxy_version=\"v1.27.2+k3s1\", module=\"TSG-OS\", module_id=\"77\", node=\"localhost\", nz_agent_id=\"3\", os_image=\"TSG-OS v23.07-rc1-079e964 (TSGXNXR620G40R01P0906)\", pod_cidr=\"10.42.0.0/24\", project=\"TSG-OS\", project_id=\"7\", provider_id=\"k3s://localhost\", service=\"prometheus-kube-state-metrics\", system_uuid=\"4c4c4544-0048-4b10-804c-b3c04f5a4233\"}" timestamp=1689547000034
Jul 17 03:37:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:42.693Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_storeFileSize\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"a8203691ca1400dbe5974df9484d373d\", table=\"default\", type=\"data\"}" timestamp=1689546999757
Jul 17 03:37:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:42.839Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546991647
Jul 17 03:37:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:42.845Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_owner\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", owner_is_controller=\"true\", owner_kind=\"Deployment\", owner_name=\"configmap-reloader\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"configmap-reloader-6c55bbc49f\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689547000034
Jul 17 03:37:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:42.908Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_threads_max\", asset=\"pcap-tsgx01\", asset_id=\"71\", container=\"metrics-server\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546985654
Jul 17 03:37:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:43.008Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_cpu_seconds_total\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-logind.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689546995764
Jul 17 03:37:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:43.051Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"rtkit-daemon.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"service\"}" timestamp=1689546995764
Jul 17 03:37:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:43.054Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", cpu=\"15\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546996242
Jul 17 03:37:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:43.090Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"100\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546991647
Jul 17 03:37:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:43.101Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-netns-cni\\\\x2d095c35f1\\\\x2d6f6e\\\\x2de7da\\\\x2db26d\\\\x2d213a1a44d8f5.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689546995764
Jul 17 03:37:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:43.144Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_address_assign_type\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"br_dign_s\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546996242
Jul 17 03:37:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:43.193Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_info\", address=\"bc:97:e1:d3:39:d3\", asset=\"pcap-tsgx01\", asset_id=\"71\", broadcast=\"ff:ff:ff:ff:ff:ff\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"eno36np3\", duplex=\"unknown\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operstate=\"down\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546996242
Jul 17 03:37:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:43.198Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db183\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546993827
Jul 17 03:37:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:43.265Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2dpartlabel-ONIE\\\\x2dBOOT.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689547000671
Jul 17 03:37:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:43.267Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_10_194_1_metric_compactionOutputSize_num_ops\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546999757
Jul 17 03:37:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:43.299Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"49\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546991647
Jul 17 03:37:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:43.803Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db231\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546993827
Jul 17 03:37:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:43.858Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-pod8e8e0871_84d7_40cc_aa16_976ff516a351.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"slice\"}" timestamp=1689546995764
Jul 17 03:37:44 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:44.057Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_LOG_metric_putTime_max\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689546999757
Jul 17 03:37:44 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:44.071Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_system_table_bucket_meta_metric_compactionTime_min\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689546999757
Jul 17 03:37:44 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:44.454Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_type\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"k3s-serving\", service=\"prometheus-kube-state-metrics\", type=\"kubernetes.io/tls\"}" timestamp=1689546995154
Jul 17 03:37:45 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:45.254Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_labels\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"tsg-traffic-engine-numa-1-firewall\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546995154
Jul 17 03:37:45 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:45.275Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_annotations\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"metrics-server-54dc485875-dfwgq\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"d1cd38b9-775b-4294-ac65-580f7b9cb70e\"}" timestamp=1689546995154
Jul 17 03:37:45 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:45.416Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_max_usage_bytes\", asset=\"pcap-tsgx01\", asset_id=\"71\", container=\"kube-multus\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546973686
Jul 17 03:37:45 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:45.466Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"pcap-tsgx01\", asset_id=\"71\", condition=\"false\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-alertmanager-6df7f457b6-m8cl7\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"98daa3ea-45f0-41c2-be94-512fb3e2a016\"}" timestamp=1689546995154
Jul 17 03:37:45 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:45.593Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", cpu=\"49\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546996242
Jul 17 03:37:45 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:45.624Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", cpu=\"86\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546996242
Jul 17 03:37:45 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:45.662Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_packets_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"vetha1f653f3\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546996242
Jul 17 03:37:45 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:45.696Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2dpartuuid-ac74cea9\\\\x2d8bbf\\\\x2d488a\\\\x2d95d2\\\\x2d9d2f22d66460.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689547000671
Jul 17 03:37:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:49.373Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_1_metric_majorCompactionTime_max\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689547007438
Jul 17 03:37:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:49.451Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_2_metric_compactionInputFileCount_99th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689547007438
Jul 17 03:37:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:49.795Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_2_metric_putTime_75th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689547007438
Jul 17 03:37:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:50.109Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_fifo_total\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens6f1np1\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547001797
Jul 17 03:37:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:50.306Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Bulkload_count\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Server\"}" timestamp=1689547007438
Jul 17 03:37:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:50.380Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_readRequestCount\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"proxy_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"12c8bac854969dda8707b0a6c52163c3\", table=\"index_time_proxy_hos_bucket\", type=\"index\"}" timestamp=1689547007438
Jul 17 03:37:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:50.671Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_metadata_generation\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"systemd-exporter-7f549d9485\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546999668
Jul 17 03:37:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:51.352Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-veth43988c23.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689547001057
Jul 17 03:37:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:51.444Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_daemonset_labels\", asset=\"twa-tsgx08\", asset_id=\"105\", daemonset=\"kube-multus-ds\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689547001568
Jul 17 03:37:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:51.683Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"coredns\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Async\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546993737
Jul 17 03:37:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:51.699Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx10\", asset_id=\"109\", cpu=\"113\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547001797
Jul 17 03:37:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:51.721Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_mod_info\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"github.com/golang/protobuf\", nz_agent_id=\"3\", program=\"systemd_exporter\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", version=\"v1.3.5\"}" timestamp=1689547001057
Jul 17 03:37:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:51.732Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_inodes_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/var/lib/kubelet/pods/2cfe23ce-7e71-44dc-8e19-b936dd7c7f8c/volumes/kubernetes.io~projected/kube-api-access-msvmx\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689547004942
Jul 17 03:37:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:51.735Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_annotations\", asset=\"twa-tsgx10\", asset_id=\"109\", configmap=\"sce-tsg-traffic-engine-numa-1\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689546999668
Jul 17 03:37:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:51.764Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_bytes_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", interface=\"cnibr0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689547006496
Jul 17 03:37:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:51.800Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_frequency_min_hertz\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"25\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547006709
Jul 17 03:37:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:51.848Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_ready\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"dign-server-web\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"dign-server-9wqkg\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"7b43283a-781d-45c1-8760-e8b3248f8fd2\"}" timestamp=1689546999668
Jul 17 03:37:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:51.861Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_allocator_allocated_bytes\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689547006625
Jul 17 03:37:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:51.875Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_nf_conntrack_entries_limit\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547001797
Jul 17 03:37:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:51.898Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.grpc.v1.cri-sandboxes-bd6a6d79498bfcaf6a81375c57beef89010d52a39f0aa9bbac6788d9facc978e-shm.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689547001057
Jul 17 03:37:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:51.904Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_frequency_min_hertz\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"16\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547006709
Jul 17 03:37:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:51.924Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:e0-0000:e0:03.1-0000:e2:00.0-net-ens8f0np0.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689547001057
Jul 17 03:37:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:51.960Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_restarts_total\", asset=\"twa-tsgx08\", asset_id=\"105\", container=\"mrapm-device\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"mrzcpd-exporter-5cf7f967cc-b62w2\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"c4ec4f52-224c-4f5b-9432-db80b6600279\"}" timestamp=1689547001568
Jul 17 03:37:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:52.140Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_packets_dropped_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", interface=\"br_dign_s\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689547006496
Jul 17 03:37:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:52.153Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", phase=\"Running\", pod=\"ipmi-exporter-6f8464dd94-96ht4\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"02feffa3-0f34-4a13-8c7e-cf66f7d181c4\"}" timestamp=1689547001568
Jul 17 03:37:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:52.514Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_mem_clients_slaves\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546999367
Jul 17 03:37:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:52.835Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", cpu=\"119\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547009337
Jul 17 03:37:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:52.904Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_multicast_packets_transmitted_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"mlx5_3\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547009337
Jul 17 03:37:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:52.956Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_down_changes_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth36d3ae80\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547009337
Jul 17 03:37:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:53.011Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-machine-id-commit.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"oneshot\", type=\"service\"}" timestamp=1689547005496
Jul 17 03:37:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:53.042Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-05cea850\\\\x2dfebe\\\\x2d4ad9\\\\x2da01e\\\\x2dde2d8c0e8a8b-volume\\\\x2dsubpaths-config\\\\x2dvolume-telegraf\\\\x2dsecurity-2.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"mount\"}" timestamp=1689547005496
Jul 17 03:37:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:53.187Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", cpu=\"104\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547009337
Jul 17 03:37:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:53.225Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-netns-cni\\\\x2d16b84d07\\\\x2d2fb8\\\\x2d88da\\\\x2d7dfb\\\\x2d6a74fdc288c3.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689547012135
Jul 17 03:37:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:53.363Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", cpu=\"126\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547009337
Jul 17 03:37:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:53.639Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_netstat_Udp_InDatagrams\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547009337
Jul 17 03:37:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:53.673Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_address_assign_type\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens4f3np3\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547009337
Jul 17 03:37:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:53.719Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_tasks_max\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-update-utmp-runlevel.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689547005496
Jul 17 03:37:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:53.786Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_packets_total\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"cadvisor\", interface=\"eno35np2\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546991284
Jul 17 03:37:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:53.800Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db35\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546998659
Jul 17 03:37:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:53.837Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db145\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546999367
Jul 17 03:37:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:53.915Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"msh-tsgx12\", asset_id=\"113\", cpu=\"34\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547003391
Jul 17 03:37:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:54.033Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_restarts_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", container=\"metrics-server\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"metrics-server-54dc485875-pqvd7\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"aed8aa53-e603-467c-8ed7-d385c606efb7\"}" timestamp=1689547004478
Jul 17 03:37:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:54.056Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_hwmon_temp_celsius\", asset=\"pcap-tsgx11\", asset_id=\"115\", chip=\"pci0000:00_0000:00:19_3\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", sensor=\"temp4\", service=\"prometheus-node-exporter\"}" timestamp=1689547009337
Jul 17 03:37:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:54.092Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_ulimits_soft\", asset=\"msh-tsgx12\", asset_id=\"113\", container=\"coredns\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", ulimit=\"max_open_files\"}" timestamp=1689546991048
Jul 17 03:37:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:54.099Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"pcap-dellr740-dt002\", asset_id=\"33\", cpu=\"2\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"NC-Gohangout-pcap-dellr740-dt002\", endpoint_id=\"173\", module=\"DC-Gohangout\", module_id=\"35\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689547012666
Jul 17 03:37:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:54.111Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_files_free\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"shm\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/dcf1f436dbba4e3531c32014df115a4bbed7f426eed73ab6c49b92cc95f20326/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547009337
Jul 17 03:37:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:54.298Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Tcp_Close_byK\", app_name=\"sapp4\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", vsysId=\"1\"}" timestamp=1689547007622
Jul 17 03:37:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:54.392Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db137\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689547010108
Jul 17 03:37:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:54.409Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sshd-keygen@ecdsa.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"service\"}" timestamp=1689547005496
Jul 17 03:37:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:54.440Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"pcap-dellr740-dt002\", asset_id=\"33\", cpu=\"34\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"NC-Gohangout-pcap-dellr740-dt002\", endpoint_id=\"173\", module=\"DC-Gohangout\", module_id=\"35\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689547012666
Jul 17 03:37:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:54.460Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-ens1f0np0.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689547012135
Jul 17 03:37:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:54.474Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_swap\", asset=\"msh-tsgx12\", asset_id=\"113\", container=\"prometheus-server\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546994694
Jul 17 03:37:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:54.574Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_cache\", asset=\"msh-tsgx12\", asset_id=\"113\", container=\"telegraf\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689546993735
Jul 17 03:37:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:54.612Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-pod17060853_6cdd_48b9_836a_8dba7f276cf9.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"slice\"}" timestamp=1689547000884
Jul 17 03:37:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:54.619Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_heap_objects\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"ipmi-exporter\"}" timestamp=1689547005070
Jul 17 03:37:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:54.683Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_speed_bytes\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"bond0\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547009337
Jul 17 03:37:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:54.686Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_created\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"tsg-traffic-engine-numa-0-firewall\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689547008950
Jul 17 03:37:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:55.053Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-dellr740-dt002\", asset_id=\"33\", cpu=\"34\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"NC-Gohangout-pcap-dellr740-dt002\", endpoint_id=\"173\", mode=\"system\", module=\"DC-Gohangout\", module_id=\"35\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689547012666
Jul 17 03:37:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:55.079Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_reservation_limit_bytes\", asset=\"msh-tsgx12\", asset_id=\"113\", container=\"dign-client\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689547000589
Jul 17 03:37:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:55.130Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_frees_total\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689546998659
Jul 17 03:37:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:55.151Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_labels\", asset=\"msh-tsgx12\", asset_id=\"113\", configmap=\"chart-content-tsg-traffic-engine-numa-1\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689547008663
Jul 17 03:37:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:55.185Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_info\", asset=\"pcap-tsgx07\", asset_id=\"85\", created_by_kind=\"Job\", created_by_name=\"helm-install-cm-policy-local-cache\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", host_ip=\"10.10.10.167\", host_network=\"false\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", node=\"localhost\", nz_agent_id=\"3\", pod=\"helm-install-cm-policy-local-cache-tpwbz\", pod_ip=\"10.42.0.68\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"25f9128d-9e49-4c7b-b03c-90b4160244e3\"}" timestamp=1689547002058
Jul 17 03:37:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:55.201Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_avail_bytes\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"shm\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/2fa54e99b592a2324df2f3fe0fa3f21476666e6b107d47b83794bffeb70d8c9f/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546999409
Jul 17 03:37:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:55.261Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_cpu_period\", asset=\"msh-tsgx08\", asset_id=\"65\", container=\"dashboard-metrics-scraper\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689546997907
Jul 17 03:37:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:55.273Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"50\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689547003391
Jul 17 03:37:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:55.280Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-ipmi-exporter-c8lpk\", project=\"TSG-OS\", project_id=\"7\", reason=\"Shutdown\", service=\"prometheus-kube-state-metrics\", uid=\"67e73254-db7c-47ad-9003-f659f9d977d7\"}" timestamp=1689547008950
Jul 17 03:37:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:55.293Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx12\", asset_id=\"113\", cpu=\"19\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547003391
Jul 17 03:37:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:55.295Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"shm\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/576bb9b5de0eb33299260ae30d648f0f1c9336a915fee8eaa5564cb35d25ae31/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547008328
Jul 17 03:37:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:55.313Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db63\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689547010108
Jul 17 03:37:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:55.452Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas_available\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"traefik\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689547002058
Jul 17 03:37:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:55.495Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_bytes_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"virtio_dign_c\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546999409
Jul 17 03:37:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:55.512Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_last_seen\", asset=\"pcap-tsgx07\", asset_id=\"85\", container=\"firewall\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689547010625
Jul 17 03:37:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:55.529Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_created\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"tsg-traffic-engine-numa-0-firewall-7fb688b4d7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689547002058
Jul 17 03:37:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:55.538Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_commands_rejected_calls_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cmd=\"zrangebyscore\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689547005184
Jul 17 03:37:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:55.543Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_flags\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"vethe9e8153e\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546999409
Jul 17 03:37:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:55.546Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_multicast_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens2f0np0\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689546999409
Jul 17 03:37:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:55.565Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"blk-availability.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"service\"}" timestamp=1689547000884
Jul 17 03:37:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:56.319Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"99\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547005883
Jul 17 03:37:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:56.325Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_swap_limit_bytes\", asset=\"pcap-tsgx03\", asset_id=\"73\", container=\"redis\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689547011563
Jul 17 03:37:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:56.335Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_resident_memory_bytes\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-udevd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689547003372
Jul 17 03:37:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:56.344Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", pod=\"kubernetes-dashboard-66dd5b456-kl6gh\", project=\"TSG-OS\", project_id=\"7\", reason=\"Shutdown\", service=\"prometheus-kube-state-metrics\", uid=\"95e7c99f-2ebd-42b8-abe6-583158385ffe\"}" timestamp=1689547013389
Jul 17 03:37:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:56.367Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_rate_bytes_per_second\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"mlx5_8\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547005883
Jul 17 03:37:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:56.837Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-infiniband-umad6.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689547003372
Jul 17 03:37:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:57.937Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_timex_pps_stability_hertz\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547005883
Jul 17 03:37:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:57.954Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db125\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689547006475
Jul 17 03:37:58 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:58.020Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_sockets\", asset=\"pcap-tsgx03\", asset_id=\"73\", container=\"kube-state-metrics\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689547008284
Jul 17 03:37:58 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:58.063Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_address_not_ready\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_endpoint=\"kubernetes-dashboard\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689547013389
Jul 17 03:37:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:37:59.696Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_commands_duration_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cmd=\"zremrangebyscore\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689547005071
Jul 17 03:38:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:00.062Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_rss\", asset=\"msh-tsgx01\", asset_id=\"51\", container=\"dign-server-web\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689547003269
Jul 17 03:38:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:00.164Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"overlay\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-4fed3108aa42b99cc67026a1e961df0207e336796f5fe373c0374ff1af88b33c-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689547004879
Jul 17 03:38:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:00.247Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_restarts_total\", asset=\"msh-tsgx01\", asset_id=\"51\", container=\"helm\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-tsg-traffic-engine-numa-1-jg62v\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"9fb763c6-2ea9-4064-be90-c60d57c27d4f\"}" timestamp=1689547007425
Jul 17 03:38:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:00.265Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"99\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689547009083
Jul 17 03:38:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:00.352Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_labels\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.traefik.v1\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689547007425
Jul 17 03:38:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:01.033Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_created\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"shaping-announce-port-tsg-traffic-engine-numa-0\", uid=\"27ba0681-5d80-4abf-a84a-b929e71680ad\"}" timestamp=1689547007425
Jul 17 03:38:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:01.096Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_labels\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-master\", uid=\"997634d2-8d57-48c3-8166-61e6bfa0882c\"}" timestamp=1689547007425
Jul 17 03:38:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:01.116Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"87\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547009083
Jul 17 03:38:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:01.139Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"scrape_samples_post_metric_relabeling\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_instance=\"10.42.1.125:8080\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689547007425
Jul 17 03:38:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:01.161Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"chronyd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"service\"}" timestamp=1689547004879
Jul 17 03:38:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:01.335Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_hwmon_chip_names\", asset=\"msh-tsgx01\", asset_id=\"51\", chip=\"pci0000:00_0000:00:18_3\", chip_name=\"k10temp\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547009083
Jul 17 03:38:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:01.350Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_netstat_IpExt_OutOctets\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547009083
Jul 17 03:38:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:01.661Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2dpath-pci\\\\x2d0000:81:00.0\\\\x2dscsi\\\\x2d0:2:0:0\\\\x2dpart3.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689547004879
Jul 17 03:38:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:02.645Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_vmstat_pgpgin\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547018318
Jul 17 03:38:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:02.700Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-sysctl.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"service\"}" timestamp=1689547008447
Jul 17 03:38:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:02.776Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_errs_total\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens8f1np1\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547018318
Jul 17 03:38:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:02.853Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_labels\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"kubernetes-dashboard\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689547011412
Jul 17 03:38:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:02.865Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"prometheus-alertmanager\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689547011412
Jul 17 03:38:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:02.900Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx02\", asset_id=\"63\", cpu=\"99\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547018318
Jul 17 03:38:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:03.010Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx02\", asset_id=\"63\", container=\"configmap-reloader\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Discard\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689547008853
Jul 17 03:38:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:03.025Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx02\", asset_id=\"63\", cpu=\"124\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547018318
Jul 17 03:38:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:03.046Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"tmpfs\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/data/var/lib/kubelet/pods/0b1be0e3-e71c-4f3b-83a7-1d29910105b8/volumes/kubernetes.io~projected/kube-api-access-b6dlx\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547018318
Jul 17 03:38:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:03.087Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db35\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689547009269
Jul 17 03:38:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:03.107Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link\", asset=\"pcap-dellr740-dt002\", asset_id=\"33\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"em1\", endpoint=\"OLAP-Node-Exporter-pcap-dellr740-dt002\", endpoint_id=\"253\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689547022497
Jul 17 03:38:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:03.139Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"msh-tsgx02\", asset_id=\"63\", container=\"kube-state-metrics\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"uninterruptible\"}" timestamp=1689547000395
Jul 17 03:38:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:03.812Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:a0-0000:a0:03.1-0000:a1:00.0-infiniband_mad-umad6.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689547008447
Jul 17 03:38:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:03.897Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"tsg_master_T_success_log\", app_name=\"tsg_master\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"traffic-engine\", line_name=\"sum\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", table_name=\"sendlog\", vsysId=\"1\"}" timestamp=1689547013892
Jul 17 03:38:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:04.232Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"nss-lookup.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"target\"}" timestamp=1689547008447
Jul 17 03:38:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:04.277Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_status_load_balancer_ingress\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", ip=\"10.10.20.162\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"traefik\", uid=\"1a2acd8b-bab5-4447-8944-5478041ae4a8\"}" timestamp=1689547011412
Jul 17 03:38:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:04.300Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_file_descriptors\", asset=\"msh-tsgx02\", asset_id=\"63\", container=\"kube-multus\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689547000237
Jul 17 03:38:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:04.356Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_scrape_collector_duration_seconds\", asset=\"msh-tsgx02\", asset_id=\"63\", collector=\"stat\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547018318
Jul 17 03:38:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:04.363Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx02\", asset_id=\"63\", cpu=\"13\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547018318
Jul 17 03:38:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:04.408Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_address_assign_type\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"vethf7fadc0b\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547018318
Jul 17 03:38:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:04.941Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"net_conntrack_dialer_conn_failed_total\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", dialer_name=\"225\", instance=\"10.10.20.159:10090\", reason=\"resolution\", type=\"global\"}" timestamp=1689547022989
Jul 17 03:38:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:04.972Z caller=dedupe.go:112 component=remote level=info remote_name=52db6f url=http://10.10.10.159:10090/nz-agent/prometheus/proxy/api/v1/write msg="Dropped sample for series that was not explicitly dropped via relabelling" ref=4436650
Jul 17 03:38:05 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:05.662Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"java_lang_MemoryPool_CollectionUsageThresholdExceeded\", asset=\"msh-dellr740-olap005\", asset_id=\"19\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Yarn-msh-dellr740-olap005-NodeManager\", endpoint_id=\"373\", module=\"NC-Yarn\", module_id=\"33\", name=\"PS Survivor Space\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689547024684
Jul 17 03:38:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:06.598Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_info\", asset=\"twa-tsgx01\", asset_id=\"91\", container=\"dign-server-dns\", container_id=\"containerd://fff17d491b5fa5b536a86b2d93293b310ac8eb642c5ca1d82880eeed7ba55573\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", image=\"docker.io/library/dign-server-dns:latest\", image_id=\"sha256:6987ac49c2d264e9289120b41e4a48536092ce6fbe41c981cdd4ad115feae8a4\", image_spec=\"dign-server-dns:latest\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"dign-server-znxnp\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"7b4d8362-f626-49d3-b132-7bb684fd42a1\"}" timestamp=1689547021227
Jul 17 03:38:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:06.637Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Succeeded\", pod=\"helm-install-ipmi-exporter-n5l8f\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"b0fbdc3c-7bf3-43b8-8574-66286b27a6a0\"}" timestamp=1689547021227
Jul 17 03:38:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:06.857Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_hwmon_temp_celsius\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", chip=\"platform_coretemp_0\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Spark-msh-dellr740-olap003\", endpoint_id=\"13\", module=\"NC-Spark\", module_id=\"47\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sensor=\"temp14\"}" timestamp=1689547026092
Jul 17 03:38:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:07.565Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", cpu=\"40\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Spark-msh-dellr740-olap003\", endpoint_id=\"13\", mode=\"nice\", module=\"NC-Spark\", module_id=\"47\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689547026092
Jul 17 03:38:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:07.704Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_memory_CommitLimit_bytes\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547023910
Jul 17 03:38:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:07.740Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cpu=\"73\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547023910
Jul 17 03:38:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:08.173Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"tmpfs\", name=\"run-k3s-containerd-io.containerd.grpc.v1.cri-sandboxes-773f634ef002e77afc003e5440e1165db1ea1674b52416a01d116bc1bbbe6302-shm.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689547013582
Jul 17 03:38:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:08.375Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cpu=\"89\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547023910
Jul 17 03:38:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:10.208Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_maxCompactionQueueSize\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"troubleshooting_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"9d47aadd56b020fb2f74604b5e408368\", table=\"index_time_troubleshooting_hos_bucket\", type=\"index\"}" timestamp=1689547028804
Jul 17 03:38:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:10.236Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_writeRequestCount\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"41801e02474f23036d0c22fbb9f5902e\", table=\"session_record_hos_bucket\", type=\"data\"}" timestamp=1689547028804
Jul 17 03:38:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:11.410Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_4_metric_majorCompactionOutputFileCount_90th_percentile\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689547030147
Jul 17 03:38:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:11.431Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_writeRequestCount\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"troubleshooting_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"85895fbc8b61b2aea3a479507705491f\", table=\"index_filename_troubleshooting_hos_bucket\", type=\"index\"}" timestamp=1689547028804
Jul 17 03:38:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:11.448Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_SYSTEM_table_MUTEX_metric_deleteBatchTime_max\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689547030147
Jul 17 03:38:11 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:11.481Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_maxFlushQueueSize\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"firewall_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"25eeeb642a8f4f09df9d4f773c55d04f\", table=\"index_filename_firewall_hos_bucket\", type=\"index\"}" timestamp=1689547028804
Jul 17 03:38:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:12.278Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_index_time_hostest_192_168_20_194_4_metric_majorCompactionInputFileCount_mean\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689547030147
Jul 17 03:38:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:12.307Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_compactionsQueuedCount\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"knowledge_base_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"7369ecb3a98d4e8c50833b5e5fd43b7e\", table=\"index_partfile_knowledge_base_hos_bucket\", type=\"index\"}" timestamp=1689547030147
Jul 17 03:38:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:12.325Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_session_record_hos_bucket_metric_compactionInputFileCount_median\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689547030147
Jul 17 03:38:12 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:12.453Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_l1CacheMissRatio\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt001-HregionServer\", endpoint_id=\"147\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Server\"}" timestamp=1689547030147
Jul 17 03:38:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:15.223Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx07\", asset_id=\"103\", container=\"dign-client\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Write\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689547020781
Jul 17 03:38:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:15.237Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_ready\", asset=\"twa-tsgx07\", asset_id=\"103\", container=\"kubernetes-dashboard\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", pod=\"kubernetes-dashboard-66dd5b456-xp4jt\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"4e581f4d-f643-47e4-804b-dd83bbeb1537\"}" timestamp=1689547028076
Jul 17 03:38:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:15.308Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_labels\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"kubernetes-dashboard-certs\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689547028076
Jul 17 03:38:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:15.321Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_address_not_ready\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_endpoint=\"prometheus-kube-state-metrics\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689547028076
Jul 17 03:38:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:15.338Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"twa-tsgx07\", asset_id=\"103\", cpu=\"117\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547021230
Jul 17 03:38:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:15.353Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_unicast_packets_transmitted_total\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_4\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547021230
Jul 17 03:38:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:15.368Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_drop_total\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethe34782c1\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547021230
Jul 17 03:38:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:15.459Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"twa-tsgx07\", asset_id=\"103\", container=\"exporter\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"sleeping\"}" timestamp=1689547012701
Jul 17 03:38:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:15.468Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_virtual_memory_max_bytes\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"smartd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689547027046
Jul 17 03:38:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:15.591Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"selinux-autorelabel-mark.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689547027046
Jul 17 03:38:15 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:15.792Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"88\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689547021230
Jul 17 03:38:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:16.126Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"111\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689547021230
Jul 17 03:38:16 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:16.927Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"usr_tx_drop_total\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", dev=\"virtio_dign_c\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"mrzcpd-exporter\"}" timestamp=1689547026692
Jul 17 03:38:17 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:17.151Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_load_average_10s\", asset=\"twa-tsgx07\", asset_id=\"103\", container=\"exporter-merger\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689547028887
Jul 17 03:38:17 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:17.368Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_errors_total\", asset=\"twa-tsgx07\", asset_id=\"103\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"cadvisor\", interface=\"ens2f0np0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689547024141
Jul 17 03:38:17 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:17.395Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_user_seconds_total\", asset=\"twa-tsgx07\", asset_id=\"103\", container=\"prometheus-alertmanager\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx07\", endpoint_id=\"455\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689547026967
Jul 17 03:38:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:19.602Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_annotations\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-tsg-traffic-engine-numa-0-sfc4c\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"02e61820-d9ba-4a99-8e7c-ef3b109dc48f\"}" timestamp=1689547033949
Jul 17 03:38:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:19.689Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_owner\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", owner_is_controller=\"true\", owner_kind=\"Job\", owner_name=\"helm-install-traefik-crd\", pod=\"helm-install-traefik-crd-6hp8h\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"e4381804-2831-4c31-a46e-78f802625973\"}" timestamp=1689547033949
Jul 17 03:38:19 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:19.957Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"crond.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689547024356
Jul 17 03:38:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:20.321Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"14\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547028263
Jul 17 03:38:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:20.323Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_tasks_max\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"udisks2.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689547030657
Jul 17 03:38:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:20.396Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dracut-pre-mount.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689547024356
Jul 17 03:38:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:20.415Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-vethef8c731a.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689547024356
Jul 17 03:38:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:20.473Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_drop_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens4f3np3\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547028263
Jul 17 03:38:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:20.503Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"76\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547028263
Jul 17 03:38:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:20.522Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"packagekit.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"service\"}" timestamp=1689547030657
Jul 17 03:38:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:20.604Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-virtual-net-veth946c4097.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689547024356
Jul 17 03:38:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:20.635Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_virtual_memory_max_bytes\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dbus.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689547030657
Jul 17 03:38:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:20.699Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"twa-tsgx06\", asset_id=\"101\", cpu=\"118\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547037612
Jul 17 03:38:20 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:20.712Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-9435b5376c50e459083851c857cbb78d7341f65d486c83f46d4c9810150bb88b.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"scope\"}" timestamp=1689547030657
Jul 17 03:38:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:21.021Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"rescue.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"target\"}" timestamp=1689547030657
Jul 17 03:38:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:21.064Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx06\", asset_id=\"101\", container=\"dashboard-metrics-scraper\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Sync\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689547027214
Jul 17 03:38:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:21.067Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-netns-cni\\\\x2d0d41555c\\\\x2d07b0\\\\x2d9271\\\\x2db0ed\\\\x2d5c43ad439589.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"mount\"}" timestamp=1689547030657
Jul 17 03:38:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:21.105Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_waiting\", asset=\"pcap-tsgx10\", asset_id=\"89\", container=\"helm\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-systemd-exporter-tfnh6\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"8506c904-444f-4def-9710-0a25e3db52fe\"}" timestamp=1689547033828
Jul 17 03:38:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:21.106Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-ask-password-console.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689547030657
Jul 17 03:38:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:21.119Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"118\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689547028263
Jul 17 03:38:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:21.151Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth4df73582\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547028263
Jul 17 03:38:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:21.167Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_compressed_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens2f0np0\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547028263
Jul 17 03:38:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:21.214Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"104\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547028263
Jul 17 03:38:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:21.234Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_speed_bytes\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens4f1np1\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547037612
Jul 17 03:38:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:21.250Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_terminated\", asset=\"twa-tsgx06\", asset_id=\"101\", container=\"prometheus-alertmanager-configmap-reload\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-alertmanager-6df7f457b6-j889p\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"da67439d-61e0-42b8-80b6-55a738bae0d9\"}" timestamp=1689547033949
Jul 17 03:38:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:21.256Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_link_mode\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethdd9c27e1\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547037612
Jul 17 03:38:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:21.270Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db181\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689547031998
Jul 17 03:38:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:21.280Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547028263
Jul 17 03:38:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:21.295Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-ens4f0np0.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689547030657
Jul 17 03:38:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:21.340Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_info\", address=\"1e:67:88:a8:81:0e\", asset=\"twa-tsgx06\", asset_id=\"101\", broadcast=\"ff:ff:ff:ff:ff:ff\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth7eb6ef28\", duplex=\"full\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operstate=\"up\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547037612
Jul 17 03:38:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:21.354Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx06\", asset_id=\"101\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx06\", endpoint_id=\"453\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"nss-user-lookup.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"target\"}" timestamp=1689547024356
Jul 17 03:38:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:21.408Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_fifo_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth4c680714\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547028263
Jul 17 03:38:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:21.421Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"msh-tsgx07\", asset_id=\"53\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx07\", endpoint_id=\"421\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"1\", pod=\"helm-install-traefik-zklzb\", project=\"TSG-OS\", project_id=\"7\", reason=\"UnexpectedAdmissionError\", service=\"prometheus-kube-state-metrics\", uid=\"92b5935a-90da-4d5f-b14a-429435655334\"}" timestamp=1689547031835
Jul 17 03:38:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:21.435Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx07\", asset_id=\"53\", cpu=\"10\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx07\", endpoint_id=\"421\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547036678
Jul 17 03:38:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:21.479Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db81\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689547034840
Jul 17 03:38:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:21.524Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_start_time_seconds\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"rescue.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689547030657
Jul 17 03:38:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:21.527Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_swap_limit_bytes\", asset=\"pcap-tsgx10\", asset_id=\"89\", container=\"mrapm-device\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689547035735
Jul 17 03:38:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:21.539Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-podab53e5a7_f22e_498d_b5af_535ae7359350.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"slice\"}" timestamp=1689547030657
Jul 17 03:38:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:21.544Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_memstats_heap_objects\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"ipmi-exporter\"}" timestamp=1689547028448
Jul 17 03:38:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:21.850Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_inodes_free\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda5\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689547026338
Jul 17 03:38:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:21.865Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"25\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547028263
Jul 17 03:38:21 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:21.961Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_threads\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"ipmi-exporter\"}" timestamp=1689547028448
Jul 17 03:38:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:22.037Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_metadata_resource_version\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.tsg-traffic-engine-numa-1.v1\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689547033828
Jul 17 03:38:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:22.039Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", cpu=\"67\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547028263
Jul 17 03:38:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:22.057Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_scrape_collector_success\", asset=\"pcap-tsgx10\", asset_id=\"89\", collector=\"netstat\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547028263
Jul 17 03:38:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:22.069Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-eb9c8310\\\\x2d844c\\\\x2d4b72\\\\x2d9e0c\\\\x2dd86e20973848-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2dvbt7g.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689547030657
Jul 17 03:38:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:22.079Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-udevd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"notify\", type=\"service\"}" timestamp=1689547030657
Jul 17 03:38:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:22.091Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"tmpfs\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/data/var/lib/kubelet/pods/ee8afe42-b7fb-4840-a9ee-3d0e7212c356/volumes/kubernetes.io~projected/kube-api-access-55vzf\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547028263
Jul 17 03:38:22 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:22.130Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_colls_total\", asset=\"pcap-tsgx10\", asset_id=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens4f3np3\", endpoint=\"TSG-OS-pcap-tsgx10\", endpoint_id=\"483\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547028263
Jul 17 03:38:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:23.525Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-dellr740-olap004\", asset_id=\"17\", cpu=\"2\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Spark-msh-dellr740-olap004\", endpoint_id=\"15\", mode=\"nice\", module=\"NC-Spark\", module_id=\"47\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689547042253
Jul 17 03:38:23 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:23.769Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_frequency_max_hertz\", asset=\"msh-dellr740-olap004\", asset_id=\"17\", cpu=\"14\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"NC-Spark-msh-dellr740-olap004\", endpoint_id=\"15\", module=\"NC-Spark\", module_id=\"47\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689547042253
Jul 17 03:38:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:24.107Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"3\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547040160
Jul 17 03:38:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:24.108Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-pci0000:80-0000:80:01.2-0000:82:00.3-net-ens4f3np3.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689547038416
Jul 17 03:38:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:24.177Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"37\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547040160
Jul 17 03:38:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:24.210Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_free_bytes\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"shm\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/452a3392db9d34bcd6408b6eb74118e030c956fa9d9c5a96dfdb1efb456e7c94/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547040160
Jul 17 03:38:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:24.285Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"pcap-tsgx12\", asset_id=\"117\", condition=\"Available\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"mrzcpd-exporter\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"true\"}" timestamp=1689547031122
Jul 17 03:38:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:24.318Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"89\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547040160
Jul 17 03:38:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:24.330Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_ports\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_endpoint=\"cm-policy-local-cache-redis-headless\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", port_name=\"tcp-redis\", port_number=\"6379\", port_protocol=\"TCP\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689547040068
Jul 17 03:38:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:24.358Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda5\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", fstype=\"ext4\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/var/lib/kubelet/pods/a27a1ebc-016c-4998-a3f3-85e865dfcd16/volume-subpaths/config-volume/telegraf-security/2\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547040160
Jul 17 03:38:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:24.359Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"prometheus-alertmanager\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", failure_type=\"pgmajfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"container\"}" timestamp=1689547038551
Jul 17 03:38:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:24.360Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_fully_labeled_replicas\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"prometheus-pushgateway-bd476bcf9\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689547031122
Jul 17 03:38:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:24.377Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"60\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547040160
Jul 17 03:38:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:24.381Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"tmpfs\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/data/var/lib/kubelet/pods/ab6b1591-cfaa-4964-8fcc-a861d0862792/volumes/kubernetes.io~projected/kube-api-access-p2tkk\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547040160
Jul 17 03:38:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:24.505Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.grpc.v1.cri-sandboxes-a675f3d38358a7aaabb7854e5e18bf454618c3432aec1414f1ea31c97a2d9987-shm.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689547031102
Jul 17 03:38:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:24.621Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_scrape_collector_duration_seconds\", asset=\"pcap-tsgx12\", asset_id=\"117\", collector=\"rapl\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547040160
Jul 17 03:38:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:24.751Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"msh-tsgx09\", asset_id=\"55\", cpu=\"73\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547034790
Jul 17 03:38:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:24.784Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_configmap_labels\", asset=\"pcap-tsgx12\", asset_id=\"117\", configmap=\"kube-root-ca.crt\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-public\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689547031122
Jul 17 03:38:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:24.811Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db128\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689547041795
Jul 17 03:38:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:24.895Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"81\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547040160
Jul 17 03:38:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:24.901Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_secret_labels\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", secret=\"sh.helm.release.v1.systemd-exporter.v1\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689547031122
Jul 17 03:38:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:24.929Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_info\", address=\"d2:fb:ba:ad:f4:86\", asset=\"msh-tsgx09\", asset_id=\"55\", broadcast=\"ff:ff:ff:ff:ff:ff\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth0c6b6d72\", duplex=\"full\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operstate=\"up\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547034790
Jul 17 03:38:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:24.949Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"ipmi.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"service\"}" timestamp=1689547038416
Jul 17 03:38:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:24.966Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-burstable.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"slice\"}" timestamp=1689547038416
Jul 17 03:38:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:24.969Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"packagekit.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"service\"}" timestamp=1689547031102
Jul 17 03:38:24 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:24.977Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-veth7a7c4663.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689547031102
Jul 17 03:38:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:25.076Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"pcap-tsgx12\", asset_id=\"117\", condition=\"false\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-pushgateway-bd476bcf9-z79gb\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"8ea47e8a-bb11-4592-aee9-0458e6cbc258\"}" timestamp=1689547031122
Jul 17 03:38:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:25.092Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"104\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547040160
Jul 17 03:38:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:25.191Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_cpu_shares\", asset=\"msh-tsgx09\", asset_id=\"55\", container=\"dign-client\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689547038586
Jul 17 03:38:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:25.210Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"msh-tsgx09\", asset_id=\"55\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx09\", endpoint_id=\"423\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", phase=\"Pending\", pod=\"mrzcpd-exporter-5cf7f967cc-7hqnr\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"2925075f-964b-44d6-8ad8-9a4ac784d408\"}" timestamp=1689547040068
Jul 17 03:38:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:25.356Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_service_labels\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"metrics-server\", uid=\"a107c5a5-7eb6-40af-abb3-9a9a76089ef5\"}" timestamp=1689547031122
Jul 17 03:38:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:25.407Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_info\", asset=\"pcap-tsgx12\", asset_id=\"117\", board_id=\"MT_0000000359\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"mlx5_0\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", firmware_version=\"22.28.4000\", hca_type=\"MT4125\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547040160
Jul 17 03:38:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:25.428Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_cpu_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-pod89567228_6ae4_4ed5_8117_7627ccbebb6d.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"slice\"}" timestamp=1689547031102
Jul 17 03:38:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:25.446Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"nfs-idmapd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"forking\", type=\"service\"}" timestamp=1689547031102
Jul 17 03:38:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:25.448Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"tmpfs\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/data/var/lib/kubelet/pods/0557366f-a93d-40e0-ba86-42e3b764cacf/volumes/kubernetes.io~projected/kube-api-access-sldnv\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547040160
Jul 17 03:38:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:25.464Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"91\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547040160
Jul 17 03:38:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:25.467Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db242\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689547041795
Jul 17 03:38:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:25.479Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_limit_bytes\", asset=\"pcap-tsgx12\", asset_id=\"117\", container=\"firewall\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689547043419
Jul 17 03:38:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:25.503Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dm-event.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689547031102
Jul 17 03:38:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:25.552Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"clickhouse_os_user_time_cpu15\", asset=\"twa-dellr740-dt002\", asset_id=\"39\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"NC-Clickhouse-twa-dellr740-dt002\", endpoint_id=\"89\", module=\"NC-Clickhouse\", module_id=\"29\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689547044866
Jul 17 03:38:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:25.559Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_status_failed\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", job_name=\"helm-install-configmap-reloader\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689547031122
Jul 17 03:38:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:25.580Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"33\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547040160
Jul 17 03:38:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:25.617Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"atd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689547031102
Jul 17 03:38:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:25.628Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kafka_network_RequestMetrics_Errors_total\", asset=\"msh-dellr740-dt003\", asset_id=\"9\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-Kafka-msh-dellr740-dt003\", endpoint_id=\"97\", error=\"REQUEST_TIMED_OUT\", module=\"DC-Kafka\", module_id=\"5\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", request=\"OffsetCommit\"}" timestamp=1689547044884
Jul 17 03:38:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:25.654Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_cluster_enabled\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689547041795
Jul 17 03:38:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:25.667Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_reads_bytes_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\"}" timestamp=1689547036876
Jul 17 03:38:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:25.695Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"pcap-tsgx12\", asset_id=\"117\", condition=\"Available\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"prometheus-kube-state-metrics\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"false\"}" timestamp=1689547031122
Jul 17 03:38:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:25.705Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_net_dev_group\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens4f1np1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547040160
Jul 17 03:38:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:25.708Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_net_dev_group\", asset=\"pcap-tsgx12\", asset_id=\"117\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth168605c4\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547040160
Jul 17 03:38:25 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:25.709Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"pcap-tsgx12\", asset_id=\"117\", cpu=\"92\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx12\", endpoint_id=\"487\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547040160
Jul 17 03:38:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:26.148Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_id\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"eno36np3\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547035810
Jul 17 03:38:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:26.185Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"auth-rpcgss-module.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", service_type=\"oneshot\", type=\"service\"}" timestamp=1689547033907
Jul 17 03:38:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:26.231Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kdump.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"service\"}" timestamp=1689547033907
Jul 17 03:38:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:26.409Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.grpc.v1.cri-sandboxes-eda31d8d78c9a278bed24273dc8c3f52f6fa296de36fcab3f456c853734b5e12-shm.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689547033907
Jul 17 03:38:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:26.721Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"nvmefc-boot-connections.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689547033907
Jul 17 03:38:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:26.748Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-22272132c2da31b9ce148bbaf64f852a794dc33f9c28a5d495b535792db4e5bd-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689547033907
Jul 17 03:38:26 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:26.878Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_reservation_limit_bytes\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689547045490
Jul 17 03:38:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:27.411Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"87\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547042882
Jul 17 03:38:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:27.436Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_annotations\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", pod=\"kubernetes-dashboard-66dd5b456-2cjmj\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"fa19b2ec-5070-431e-b30c-dc1e010c4b86\"}" timestamp=1689547035682
Jul 17 03:38:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:27.463Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_created\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"prometheus-alertmanager-6df7f457b6\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689547035682
Jul 17 03:38:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:27.466Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_bytes_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"eno2\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547042882
Jul 17 03:38:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:27.486Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_tasks_max\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"k3s-orphaned-pods-dir-clean.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689547034114
Jul 17 03:38:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:27.497Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Ipv6_Bit\", app_name=\"sapp4\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", vsysId=\"1\"}" timestamp=1689547044090
Jul 17 03:38:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:27.503Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"msh-tsgx10\", asset_id=\"69\", condition=\"false\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-traefik-crd-m2nsh\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"6ee7b6d8-27b2-4118-9cf0-9229465d0b5b\"}" timestamp=1689547035682
Jul 17 03:38:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:27.571Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547042882
Jul 17 03:38:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:27.592Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"coredns\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Discard\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689547037731
Jul 17 03:38:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:27.646Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_bytes_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", interface=\"eno35np2\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689547044573
Jul 17 03:38:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:27.664Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"127\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547042882
Jul 17 03:38:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:27.750Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"coredns\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Sync\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689547037731
Jul 17 03:38:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:27.772Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_usage_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", container=\"firewall\", cpu=\"total\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689547044216
Jul 17 03:38:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:27.782Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_writes_merged_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda2\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689547035181
Jul 17 03:38:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:27.795Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_compressed_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth2e13d535\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547042882
Jul 17 03:38:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:27.796Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_cache\", asset=\"twa-tsgx09\", asset_id=\"107\", container=\"dign-client\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689547044472
Jul 17 03:38:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:27.812Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_cpu_shares\", asset=\"twa-tsgx09\", asset_id=\"107\", container=\"lb-tcp-443\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689547045490
Jul 17 03:38:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:27.822Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx09\", asset_id=\"107\", container=\"dign-client\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Async\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689547044472
Jul 17 03:38:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:27.837Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx09\", asset_id=\"107\", cpu=\"106\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547035810
Jul 17 03:38:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:27.846Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_errs_total\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth45cfb0fe\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547035810
Jul 17 03:38:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:27.847Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_commands_failed_calls_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cmd=\"expire\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689547039446
Jul 17 03:38:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:27.856Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-random-seed.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"service\"}" timestamp=1689547034114
Jul 17 03:38:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:27.862Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db62\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689547039446
Jul 17 03:38:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:27.891Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_waiting\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"metrics\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"cm-policy-local-cache-redis-master-69967b5879-fpcq2\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"78ab1f97-9ecb-47ed-ae81-048055fb4556\"}" timestamp=1689547035682
Jul 17 03:38:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:27.912Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_reads_total\", asset=\"twa-tsgx09\", asset_id=\"107\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689547044683
Jul 17 03:38:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:27.924Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"78\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547042882
Jul 17 03:38:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:27.942Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_mtu_bytes\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens4f1np1\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547042882
Jul 17 03:38:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:27.979Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_files\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda5\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", fstype=\"ext4\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/var\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547042882
Jul 17 03:38:27 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:27.990Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_info\", asset=\"twa-tsgx09\", asset_id=\"107\", container=\"ipmi-exporter\", container_id=\"containerd://8b9feda393da79cdcc022dce7dbaba9050da0b4ee5026355c0163cd6867d1f60\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx09\", endpoint_id=\"457\", exported_job=\"exporter\", image=\"docker.io/pnnlmiscscripts/ipmi-exporter:1.2.0-9\", image_id=\"sha256:69b7e13cc9877c8676efbecca48c73b88b2a0c442d08f05421bebb6b7df404e3\", image_spec=\"docker.io/pnnlmiscscripts/ipmi-exporter:1.2.0-9\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"ipmi-exporter-6f8464dd94-c9gkv\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"8abd475c-c69e-4e96-9ca6-1c2e74278a42\"}" timestamp=1689547038744
Jul 17 03:38:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:28.069Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_drop_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"br_dign_s\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547042882
Jul 17 03:38:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:28.170Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_writes_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda5\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689547035181
Jul 17 03:38:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:28.173Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_scrape_collector_success\", asset=\"msh-tsgx10\", asset_id=\"69\", collector=\"edac\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547042882
Jul 17 03:38:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:28.187Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"kube-multus\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", failure_type=\"pgfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"hierarchy\"}" timestamp=1689547034388
Jul 17 03:38:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:28.201Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", phase=\"Running\", pod=\"dign-client-l6t52\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"b60c5b05-6d97-4ae4-bf4c-866d931e73ec\"}" timestamp=1689547035682
Jul 17 03:38:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:28.219Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_ready\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"dashboard-metrics-scraper\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", pod=\"dashboard-metrics-scraper-5b94897dc9-8srnj\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"c0c07089-8d67-4a93-9c72-855e72ebd4e3\"}" timestamp=1689547035682
Jul 17 03:38:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:28.230Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-rdma-devices-mlx5_5:1-umad.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689547034114
Jul 17 03:38:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:28.241Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"ext4\", name=\"var-lib-kubelet-pods-0b0292d1\\\\x2d7c6a\\\\x2d4fad\\\\x2da1f9\\\\x2df14e3c7cb4f8-volume\\\\x2dsubpaths-config\\\\x2dvolume-telegraf-2.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689547034114
Jul 17 03:38:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:28.262Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-944d9a8d42057e643cabbf98de2f0783e9bee0ae7979e1b0c1ec1f675f2687c6.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"scope\"}" timestamp=1689547034114
Jul 17 03:38:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:28.315Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Eth_Pkt\", app_name=\"sapp4\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", vsysId=\"1\"}" timestamp=1689547046715
Jul 17 03:38:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:28.536Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"prometheus-alertmanager-configmap-reload\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", failure_type=\"pgmajfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"hierarchy\"}" timestamp=1689547034476
Jul 17 03:38:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:28.550Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_waiting\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"helm\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-tsg-traffic-engine-numa-1-7rxbb\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"03195aa0-7d32-4be5-b57f-109823710aac\"}" timestamp=1689547035682
Jul 17 03:38:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:28.558Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"101\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547042882
Jul 17 03:38:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:28.568Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_swap_limit_bytes\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"prometheus-server-configmap-reload\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689547044841
Jul 17 03:38:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:28.576Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_cpu_user_children_seconds_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689547039446
Jul 17 03:38:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:28.586Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx10\", asset_id=\"69\", cpu=\"32\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547042882
Jul 17 03:38:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:28.671Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-75dc4160\\\\x2dda0a\\\\x2d4767\\\\x2d823a\\\\x2d227b8dd303c1-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2dc4vrv.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689547034114
Jul 17 03:38:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:28.684Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"cortex_distributor_ingester_clients\", agent_name=\"msh-kvm-nz001\", application=\"loki\", datacenter=\"MSH\", instance=\"127.0.0.1:13100\"}" timestamp=1689547048327
Jul 17 03:38:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:28.724Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_restarts_total\", asset=\"msh-tsgx10\", asset_id=\"69\", container=\"dign-server-dns\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"dign-server-pf96k\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"75dc4160-da0a-4767-823a-227b8dd303c1\"}" timestamp=1689547035682
Jul 17 03:38:28 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:28.743Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_io_time_weighted_seconds_total\", asset=\"msh-tsgx10\", asset_id=\"69\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/sys/fs/cgroup\", endpoint=\"TSG-OS-msh-tsgx10\", endpoint_id=\"437\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689547035181
Jul 17 03:38:29 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:29.873Z caller=dedupe.go:112 component=remote level=error remote_name=52db6f url=http://10.10.10.159:10090/nz-agent/prometheus/proxy/api/v1/write msg="non-recoverable error" count=1000 exemplarCount=0 err="server returned HTTP status 400 : out of order sample"
Jul 17 03:38:30 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:30.941Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kafka_server_BrokerTopicMetrics_OneMinuteRate\", asset=\"twa-dellr740-dt002\", asset_id=\"39\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-Kafka-twa-dellr740-dt002\", endpoint_id=\"107\", module=\"DC-Kafka\", module_id=\"5\", name=\"TotalProduceRequestsPerSec\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", topic=\"DOS-SKETCH-RECORD\"}" timestamp=1689547049470
Jul 17 03:38:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:31.139Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.grpc.v1.cri-sandboxes-16b0c54f99bc831e5c2f40ec71018ed8402840a09fc9afa8e373fbc9e8816ba5-shm.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689547041015
Jul 17 03:38:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:31.304Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens3f1np1\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547040939
Jul 17 03:38:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:31.306Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_complete\", asset=\"msh-tsgx05\", asset_id=\"67\", condition=\"true\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", job_name=\"helm-install-ipmi-exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689547044068
Jul 17 03:38:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:31.315Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_reservation_limit_bytes\", asset=\"msh-tsgx05\", asset_id=\"67\", container=\"exporter\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689547035836
Jul 17 03:38:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:31.320Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_net_dev_group\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"eno34np1\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547040939
Jul 17 03:38:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:31.337Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-e186b759\\\\x2d1542\\\\x2d4733\\\\x2d9263\\\\x2d82c6ae7fed9e-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2db6gmg.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689547041015
Jul 17 03:38:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:31.357Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_info\", asset=\"msh-tsgx05\", asset_id=\"67\", container=\"prometheus-alertmanager-configmap-reload\", container_id=\"containerd://a27095b60f29c623fd77de735e25cdaa506cee62cd4a80d9b9fd2589d7ae78e2\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", image=\"docker.io/jimmidyson/configmap-reload:v0.5.0\", image_id=\"sha256:d771cc9785a13659c0e9363cae1d238bb58114a3340f805564daaed2494475f8\", image_spec=\"jimmidyson/configmap-reload:v0.5.0\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-alertmanager-6df7f457b6-7vmmh\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"a69122b9-4d3a-46e4-a296-9ed5e1d8b954\"}" timestamp=1689547044068
Jul 17 03:38:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:31.447Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Unknown_Pkt\", app_name=\"sapp4\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", vsysId=\"1\"}" timestamp=1689547040718
Jul 17 03:38:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:31.505Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dracut-pre-trigger.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"service\"}" timestamp=1689547046907
Jul 17 03:38:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:31.526Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_created\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"kubernetes-dashboard\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689547050436
Jul 17 03:38:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:31.620Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"var-lib-kubelet-pods-23dba7b8\\\\x2d644d\\\\x2d447e\\\\x2d8179\\\\x2d7202ec14e423-volumes-kubernetes.io\\\\x7esecret-config.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"mount\"}" timestamp=1689547046907
Jul 17 03:38:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:31.640Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_errors_received_total\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"mlx5_1\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547039093
Jul 17 03:38:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:31.642Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_writes_total\", asset=\"msh-tsgx05\", asset_id=\"67\", container=\"redis\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689547024735
Jul 17 03:38:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:31.661Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_compressed_total\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethf7cbe37a\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547039093
Jul 17 03:38:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:31.662Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"85\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689547040939
Jul 17 03:38:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:31.687Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_replicas\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"mrzcpd-exporter-5cf7f967cc\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689547039276
Jul 17 03:38:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:31.692Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_daemonset_status_current_number_scheduled\", asset=\"msh-tsgx11\", asset_id=\"111\", daemonset=\"kube-multus-ds\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689547039276
Jul 17 03:38:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:31.698Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"56\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547040791
Jul 17 03:38:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:31.702Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_speed_bytes\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth17e7614b\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547040791
Jul 17 03:38:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:31.711Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_bytes_total\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"vethdc7cf888\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547040939
Jul 17 03:38:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:31.712Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled_time\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", pod=\"tsg-traffic-engine-numa-1-firewall-6d8d5654d-wdp2k\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"506c9da6-9ad1-4f6d-afc6-e4606512a111\"}" timestamp=1689547039276
Jul 17 03:38:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:31.719Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_compressed_total\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"vethf7a1468f\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547040791
Jul 17 03:38:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:31.722Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"nfs-mountd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"service\"}" timestamp=1689547042644
Jul 17 03:38:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:31.732Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_restart_policy\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"svclb-traefik-401de602-mxwbd\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", type=\"Always\", uid=\"ed59a17b-7df5-491f-9423-c86e0048b016\"}" timestamp=1689547044068
Jul 17 03:38:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:31.748Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"2\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547040939
Jul 17 03:38:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:31.751Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"tmpfs\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/data/var/lib/kubelet/pods/d4700480-e588-4509-b70f-b20c4e312a51/volumes/kubernetes.io~projected/kube-api-access-gndz5\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547040791
Jul 17 03:38:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:31.754Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"nginx_vts_main_connections\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"Galaxy-hos-nginx-msh-dellr740-dt001\", endpoint_id=\"201\", module=\"Galaxy-hos-nginx\", module_id=\"13\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", status=\"active\"}" timestamp=1689547050872
Jul 17 03:38:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:31.787Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Tcp_Bit\", app_name=\"sapp4\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", vsysId=\"1\"}" timestamp=1689547047477
Jul 17 03:38:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:31.818Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx05\", asset_id=\"67\", cpu=\"101\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547040939
Jul 17 03:38:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:31.846Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"tmpfs\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/data/var/lib/kubelet/pods/a69122b9-4d3a-46e4-a296-9ed5e1d8b954/volumes/kubernetes.io~projected/kube-api-access-mt966\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547040939
Jul 17 03:38:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:31.883Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens4f0np0\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547040939
Jul 17 03:38:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:31.903Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"vethf7e6d432\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547040791
Jul 17 03:38:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:31.917Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_drop_total\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens1f1np1\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547040791
Jul 17 03:38:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:31.918Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_fifo_total\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens8f1np1\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547040939
Jul 17 03:38:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:31.929Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Tcp_Link_Double_ALL\", app_name=\"sapp4\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", vsysId=\"1\"}" timestamp=1689547042297
Jul 17 03:38:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:31.941Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_daemonset_metadata_generation\", asset=\"msh-tsgx11\", asset_id=\"111\", daemonset=\"kube-multus-ds\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689547039276
Jul 17 03:38:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:31.955Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_labels\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-systemd-exporter-zbfbx\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"417712f3-89ba-43e3-a520-3343986e90fc\"}" timestamp=1689547050436
Jul 17 03:38:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:31.960Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", phase=\"Unknown\", pod=\"systemd-exporter-7f549d9485-bph7s\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"758d8a04-3fb4-4af1-97bd-b2b3e1439f97\"}" timestamp=1689547039276
Jul 17 03:38:31 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:31.972Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_readonly\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"shm\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/e983fa51aa6cba88fd7de40052691c968c60a7f4c557d552f1b51456d45ff481/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547039093
Jul 17 03:38:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:32.046Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547040791
Jul 17 03:38:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:32.058Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"43\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547040791
Jul 17 03:38:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:32.062Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda5\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", fstype=\"ext4\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/data/var/lib/kubelet/pods/d8012c00-aaec-4f56-a35b-ad6f4d1ef590/volume-subpaths/config-volume/telegraf/2\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547040939
Jul 17 03:38:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:32.075Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"39\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547040791
Jul 17 03:38:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:32.091Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_address_assign_type\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"br_dign_s\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547040791
Jul 17 03:38:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:32.092Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_compressed_total\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens3f0np0\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547040791
Jul 17 03:38:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:32.107Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"dashboard-metrics-scraper\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Write\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689547029307
Jul 17 03:38:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:32.115Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx03\", asset_id=\"95\", cpu=\"86\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547039093
Jul 17 03:38:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:32.122Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"94\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547040791
Jul 17 03:38:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:32.141Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"93\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547040791
Jul 17 03:38:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:32.142Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas_updated\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"coredns\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689547044068
Jul 17 03:38:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:32.155Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"4\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547040791
Jul 17 03:38:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:32.159Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_restart_policy\", asset=\"msh-tsgx05\", asset_id=\"67\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx05\", endpoint_id=\"435\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-traefik-t9td5\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", type=\"OnFailure\", uid=\"57da4140-5f56-43ff-a801-4cf3cf20c06e\"}" timestamp=1689547044068
Jul 17 03:38:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:32.164Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"traefik-64f55bb67d-hlldq\", project=\"TSG-OS\", project_id=\"7\", reason=\"NodeLost\", service=\"prometheus-kube-state-metrics\", uid=\"9779568b-efbc-449c-9718-0ebca3e547e4\"}" timestamp=1689547050436
Jul 17 03:38:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:32.171Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-1d68b5d27a8f4e5d7f1cfba0571d0ccc472629c36a67933adc44728f3233a8d2-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689547042644
Jul 17 03:38:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:32.261Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_total_error_replies\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689547036711
Jul 17 03:38:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:32.263Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_reads_bytes_total\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"metrics\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689547027845
Jul 17 03:38:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:32.271Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_status_replicas\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"systemd-exporter-7f549d9485\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689547050436
Jul 17 03:38:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:32.273Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-platform-serial8250-tty-ttyS2.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689547042644
Jul 17 03:38:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:32.277Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failcnt\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"lb-tcp-80\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689547035445
Jul 17 03:38:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:32.285Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_readonly\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"shm\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/c395d2b9a61a4ed42882fe39bc37eb752ad86081e765e4ec68f819933869c195/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547039093
Jul 17 03:38:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:32.392Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"unbound-anchor.timer\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"timer\"}" timestamp=1689547042644
Jul 17 03:38:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:32.498Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-virtual-net-veth9955cde4.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"device\"}" timestamp=1689547046907
Jul 17 03:38:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:32.629Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"jvm_memory_bytes_used\", area=\"heap\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-Yarn-twa-dellr740-dt001-NodeManager\", endpoint_id=\"363\", module=\"DC-Yarn\", module_id=\"1\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689547051354
Jul 17 03:38:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:32.642Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"scrape_duration_seconds\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_instance=\"10.10.30.163:9008\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689547046907
Jul 17 03:38:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:32.709Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"twa-tsgx03\", asset_id=\"95\", container=\"coredns\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/dev/sda\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Write\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689547024785
Jul 17 03:38:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:32.738Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_annotations\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"metrics-server-54dc485875\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689547050436
Jul 17 03:38:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:32.765Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2duuid-DA1E\\\\x2d750B.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689547042644
Jul 17 03:38:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:32.799Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_carrier_total\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethc3a959fd\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547040605
Jul 17 03:38:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:32.877Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_bytes_total\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"eno36np3\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547040605
Jul 17 03:38:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:32.902Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"74\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547040605
Jul 17 03:38:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:32.926Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_mtu_bytes\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth914fec83\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547039093
Jul 17 03:38:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:32.932Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"cri-containerd-6a692a6c37775cfac2c8d862aac1e1c484c074fd3c884467f03541ec8697bb21.scope\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"scope\"}" timestamp=1689547047691
Jul 17 03:38:32 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:32.984Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_fifo_total\", asset=\"twa-tsgx03\", asset_id=\"95\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth86490e66\", endpoint=\"TSG-OS-twa-tsgx03\", endpoint_id=\"447\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547039093
Jul 17 03:38:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:33.018Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"111\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689547040605
Jul 17 03:38:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:33.034Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"17\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547040605
Jul 17 03:38:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:33.050Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-infiniband-umad1.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689547047691
Jul 17 03:38:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:33.338Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas_available\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"prometheus-pushgateway\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689547039276
Jul 17 03:38:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:33.372Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_running\", asset=\"msh-tsgx11\", asset_id=\"111\", container=\"lb-tcp-443\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"svclb-traefik-46c02faf-rzf44\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"59e0f471-ccc6-4ff5-8571-7fb136f39b65\"}" timestamp=1689547039276
Jul 17 03:38:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:33.387Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"100\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547040791
Jul 17 03:38:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:33.407Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx11\", asset_id=\"111\", cpu=\"81\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547040791
Jul 17 03:38:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:33.424Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_compressed_total\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"lo\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547040791
Jul 17 03:38:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:33.432Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_namespace_status_phase\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", nz_agent_id=\"3\", phase=\"Terminating\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689547039276
Jul 17 03:38:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:33.446Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_commands_processed_total\", asset=\"msh-tsgx11\", asset_id=\"111\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx11\", endpoint_id=\"439\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689547036711
Jul 17 03:38:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:33.618Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db237\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689547049260
Jul 17 03:38:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:33.635Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"rpc-statd-notify.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"service\"}" timestamp=1689547047691
Jul 17 03:38:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:33.712Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_inodes_total\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/run\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689547030085
Jul 17 03:38:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:33.794Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"53\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689547040605
Jul 17 03:38:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:33.814Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db106\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689547049260
Jul 17 03:38:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:33.881Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_namespace_status_phase\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-public\", nz_agent_id=\"3\", phase=\"Active\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689547039894
Jul 17 03:38:33 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:33.900Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"twa-tsgx05\", asset_id=\"99\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db3\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689547049260
Jul 17 03:38:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:34.024Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"twa-tsgx05\", asset_id=\"99\", condition=\"Progressing\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"metrics-server\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"unknown\"}" timestamp=1689547039894
Jul 17 03:38:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:34.052Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"twa-tsgx05\", asset_id=\"99\", condition=\"Available\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"configmap-reloader\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"true\"}" timestamp=1689547039894
Jul 17 03:38:34 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:34.070Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx05\", asset_id=\"99\", cpu=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx05\", endpoint_id=\"451\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547040605
Jul 17 03:38:37 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:37.384Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"concurrent_queries\", asset=\"msh-dellr740-olap001\", asset_id=\"11\", cluster=\"query\", cluster_node=\"192.168.20.222:8123\", cluster_user=\"tsg_query\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"Galaxy-chproxy-msh-dellr740-olap001\", endpoint_id=\"39\", module=\"Galaxy-chproxy\", module_id=\"9\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", replica=\"default\", user=\"tsg_query\"}" timestamp=1689547057291
Jul 17 03:38:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:40.373Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_endpoint_info\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_endpoint=\"cm-policy-local-cache-redis-master\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689547060034
Jul 17 03:38:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:40.456Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"82\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547051647
Jul 17 03:38:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:40.581Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"73\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689547051647
Jul 17 03:38:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:40.953Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_node_status_capacity\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", node=\"localhost\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", resource=\"cpu\", service=\"prometheus-kube-state-metrics\", unit=\"core\"}" timestamp=1689547060034
Jul 17 03:38:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:40.972Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_info\", asset=\"twa-tsgx04\", asset_id=\"97\", created_by_kind=\"ReplicaSet\", created_by_name=\"dashboard-metrics-scraper-5b94897dc9\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", host_ip=\"10.10.30.164\", host_network=\"false\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kubernetes-dashboard\", node=\"localhost\", nz_agent_id=\"3\", pod=\"dashboard-metrics-scraper-5b94897dc9-9t7mj\", pod_ip=\"10.42.0.49\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"e367f0c1-a70e-49ee-9c5c-4e518cf2740f\"}" timestamp=1689547060034
Jul 17 03:38:40 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:40.988Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"123\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547051647
Jul 17 03:38:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:41.014Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"clevis-luks-askpass.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"service\"}" timestamp=1689547055764
Jul 17 03:38:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:41.032Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas_ready\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"local-path-provisioner\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689547060034
Jul 17 03:38:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:41.159Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"32\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547051647
Jul 17 03:38:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:41.174Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_incrementCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", namespace=\"SYSTEM\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"0e32bd2946de68f3772b0f5addf5bbbf\", table=\"LOG\", type=\"data\"}" timestamp=1689547059757
Jul 17 03:38:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:41.211Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"71\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689547051647
Jul 17 03:38:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:41.247Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"process_cpu_seconds_total\", asset=\"pcap-dellr740-dt002\", asset_id=\"33\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-Yarn-pcap-dellr740-dt002-NodeManager\", endpoint_id=\"359\", module=\"DC-Yarn\", module_id=\"1\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689547061133
Jul 17 03:38:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:41.271Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_scrape_collector_success\", asset=\"twa-tsgx04\", asset_id=\"97\", collector=\"btrfs\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547051647
Jul 17 03:38:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:41.285Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Tcp_Bit\", app_name=\"sapp4\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", vsysId=\"1\"}" timestamp=1689547053536
Jul 17 03:38:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:41.305Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_max_fds\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"polkit.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689547055764
Jul 17 03:38:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:41.386Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx04\", asset_id=\"97\", cpu=\"80\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547051647
Jul 17 03:38:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:41.450Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_device_id\", asset=\"twa-tsgx04\", asset_id=\"97\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"ens3f1np1\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547051647
Jul 17 03:38:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:41.580Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failcnt\", asset=\"twa-tsgx04\", asset_id=\"97\", container=\"prometheus-alertmanager-configmap-reload\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx04\", endpoint_id=\"449\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689547048388
Jul 17 03:38:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:41.659Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_filteredReadRequestCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"rtp_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"daf3b05a3b1e3315b0712daa04db60c2\", table=\"index_filename_rtp_hos_bucket\", type=\"index\"}" timestamp=1689547059757
Jul 17 03:38:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:41.752Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_compactionsCompletedCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"c0d08dd9c8090a5d8442f6cb602de9da\", table=\"session_record_hos_bucket\", type=\"data\"}" timestamp=1689547059757
Jul 17 03:38:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:41.767Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_10_193_1_metric_flushOutputSize_98th_percentile\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689547059757
Jul 17 03:38:41 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:41.824Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_flushesQueuedCount\", asset=\"pcap-dellr740-dt001\", asset_id=\"31\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"DC-HBase-pcap-dellr740-dt001-HregionServer\", endpoint_id=\"153\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"default\", project=\"OLAP\", project_id=\"3\", region=\"d839716bb431f8ae37418c3ce87ab845\", table=\"index_partfile_default\", type=\"index\"}" timestamp=1689547059757
Jul 17 03:38:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:42.451Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_container_status_ready\", asset=\"pcap-tsgx01\", asset_id=\"71\", container=\"kube-state-metrics\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-kube-state-metrics-6b768c7bf8-dzkd6\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"50480520-32d8-413b-9354-fde3e50919a7\"}" timestamp=1689547055154
Jul 17 03:38:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:42.478Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"112\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689547056242
Jul 17 03:38:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:42.500Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_running_seconds_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", cpu=\"125\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547056242
Jul 17 03:38:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:42.771Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"k3s-orphaned-pods-dir-clean.timer\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"timer\"}" timestamp=1689547060671
Jul 17 03:38:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:42.819Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Failed\", pod=\"helm-install-traefik-crd-r4msj\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"b4b0df3d-0f85-4950-9e3b-a5a66a88bc21\"}" timestamp=1689547055154
Jul 17 03:38:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:42.891Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_timeslices_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", cpu=\"10\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547056242
Jul 17 03:38:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:42.931Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db145\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689547054838
Jul 17 03:38:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:42.955Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_usage_seconds_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", container=\"mrapm-device\", cpu=\"total\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689547058649
Jul 17 03:38:42 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:42.975Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", cpu=\"106\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547056242
Jul 17 03:38:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:43.682Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kmod-static-nodes.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"activating\", type=\"service\"}" timestamp=1689547060671
Jul 17 03:38:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:43.872Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_daemonset_status_number_ready\", asset=\"pcap-tsgx01\", asset_id=\"71\", daemonset=\"dign-server\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689547055154
Jul 17 03:38:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:43.886Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"pcap-tsgx01\", asset_id=\"71\", condition=\"unknown\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-prometheus-qzxgz\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"5bc779e9-f51a-4b57-87de-ce5b690c25f8\"}" timestamp=1689547055154
Jul 17 03:38:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:43.951Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_errs_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens6f0np0\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547056242
Jul 17 03:38:43 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:43.993Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_scrape_collector_duration_seconds\", asset=\"pcap-tsgx01\", asset_id=\"71\", collector=\"timex\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547056242
Jul 17 03:38:44 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:44.010Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"initrd-root-fs.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"target\"}" timestamp=1689547060671
Jul 17 03:38:44 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:44.065Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_usage_seconds_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", container=\"monit-stream\", cpu=\"total\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689547054230
Jul 17 03:38:44 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:44.068Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx01\", asset_id=\"71\", cpu=\"120\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547056242
Jul 17 03:38:44 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:44.086Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth7183a5ce\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547056242
Jul 17 03:38:44 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:44.121Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"pcap-tsgx01\", asset_id=\"71\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx01\", endpoint_id=\"465\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"ext4\", name=\"data-var-lib-kubelet-pods-d20d4034\\\\x2d164e\\\\x2d4450\\\\x2da851\\\\x2d056ee84ef97e-volume\\\\x2dsubpaths-config\\\\x2dvolume-telegraf-2.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689547060671
Jul 17 03:38:44 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:44.555Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"msh-kvm-cm001\", asset_id=\"3\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"CM-Node-Exporter-msh-kvm-cm001\", endpoint_id=\"391\", module=\"CM-Node-Exporter\", module_id=\"75\", name=\"14\", nz_agent_id=\"3\", parent_asset=\"msh-dellr740-pve001\", parent_asset_id=\"1\", project=\"Central-Management\", project_id=\"5\", type=\"Processor\"}" timestamp=1689547064235
Jul 17 03:38:44 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:44.583Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_memory_Percpu_bytes\", asset=\"msh-kvm-cm001\", asset_id=\"3\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"CM-Node-Exporter-msh-kvm-cm001\", endpoint_id=\"391\", module=\"CM-Node-Exporter\", module_id=\"75\", nz_agent_id=\"3\", parent_asset=\"msh-dellr740-pve001\", parent_asset_id=\"1\", project=\"Central-Management\", project_id=\"5\"}" timestamp=1689547064235
Jul 17 03:38:48 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:48.583Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_DecreaseMemStoreSize_num_ops\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Memory\"}" timestamp=1689547067439
Jul 17 03:38:48 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:48.763Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_194_3_metric_putBatchTime_min\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689547067439
Jul 17 03:38:48 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:48.852Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_tsg_galaxy_table_gtpc_knowledge_base_metric_deleteTime_num_ops\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689547067439
Jul 17 03:38:48 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:48.894Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_tsg_galaxy_table_gtpc_knowledge_base_metric_putBatchTime_num_ops\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"TableLatencies\"}" timestamp=1689547067439
Jul 17 03:38:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:49.349Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_Namespace_default_table_hostest_192_168_20_193_1_metric_majorCompactionOutputFileCount_99_9th_percentile\", asset=\"msh-dellr740-dt002\", asset_id=\"7\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"DC-HBase-msh-dellr740-dt002-HregionServer\", endpoint_id=\"149\", module=\"DC-HBase\", module_id=\"3\", name=\"RegionServer\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", sub=\"Tables\"}" timestamp=1689547067439
Jul 17 03:38:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:49.633Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"emergency.target\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"target\"}" timestamp=1689547061057
Jul 17 03:38:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:49.639Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_frame_total\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"br_dign_c\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547061797
Jul 17 03:38:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:49.668Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_daemonset_status_current_number_scheduled\", asset=\"twa-tsgx10\", asset_id=\"109\", daemonset=\"dign-client\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689547059668
Jul 17 03:38:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:49.685Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"twa-tsgx10\", asset_id=\"109\", condition=\"Progressing\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"metrics-server\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"unknown\"}" timestamp=1689547059668
Jul 17 03:38:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:49.723Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_packets_total\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth9706f3d5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547061797
Jul 17 03:38:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:49.885Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_compressed_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"vethcf0ceaf9\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547066709
Jul 17 03:38:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:49.933Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_udp_queues\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", ip=\"v4\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", queue=\"rx\", service=\"prometheus-node-exporter\"}" timestamp=1689547061797
Jul 17 03:38:49 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:49.939Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"exporter-merger\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"iowaiting\"}" timestamp=1689547047716
Jul 17 03:38:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:50.001Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-olap003\", endpoint_id=\"235\", module=\"OLAP-Node-Exporter\", module_id=\"51\", name=\"44\", nz_agent_id=\"3\", olap_node_exporter=\"yarn_resourcemanager,zookeeper,nacos,kafka,spark,druid_data,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\", type=\"Processor\"}" timestamp=1689547069713
Jul 17 03:38:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:50.015Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dracut-pre-pivot.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689547066554
Jul 17 03:38:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:50.075Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"rdma-ndd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"service\"}" timestamp=1689547061057
Jul 17 03:38:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:50.093Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_resident_memory_bytes\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"exporter-systemd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689547066554
Jul 17 03:38:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:50.133Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth0f90183e\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547061797
Jul 17 03:38:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:50.248Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_io_time_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"overlay_0-257\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689547065286
Jul 17 03:38:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:50.264Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_failures_total\", asset=\"twa-tsgx08\", asset_id=\"105\", container=\"prometheus-alertmanager\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", failure_type=\"pgmajfault\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", scope=\"hierarchy\"}" timestamp=1689547060155
Jul 17 03:38:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:50.337Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Unknown\", pod=\"helm-install-cm-policy-local-cache-rsnqh\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"f1dadae1-a545-435a-9bce-9a35d9db87be\"}" timestamp=1689547061568
Jul 17 03:38:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:50.351Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"45\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547066709
Jul 17 03:38:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:50.370Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_drop_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth2ffdece6\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547066709
Jul 17 03:38:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:50.425Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_reservation_limit_bytes\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"dign-server-dns\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689547059050
Jul 17 03:38:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:50.444Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_condition\", asset=\"twa-tsgx08\", asset_id=\"105\", condition=\"Progressing\", datacenter=\"TWA\", datacenter_id=\"5\", deployment=\"tsg-traffic-engine-numa-1-firewall\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", status=\"false\"}" timestamp=1689547061568
Jul 17 03:38:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:50.460Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_cpu_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-burstable-podfd4b4291_55db_4894_9795_6abc7f3024db.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"slice\"}" timestamp=1689547066554
Jul 17 03:38:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:50.475Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_ulimits_soft\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"telegraf\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\", ulimit=\"max_open_files\"}" timestamp=1689547051173
Jul 17 03:38:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:50.533Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_inodes_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/data/var/lib/kubelet/pods/e4e2672a-a045-406b-b912-a701b409de63/volumes/kubernetes.io~projected/kube-api-access-qxdsd\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689547065286
Jul 17 03:38:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:50.540Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"126\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", mode=\"iowait\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547066709
Jul 17 03:38:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:50.548Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_labels\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"helm-install-traefik-crd-h4dtp\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"440a9824-0943-4c19-88b9-06b635f437bf\"}" timestamp=1689547059668
Jul 17 03:38:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:50.570Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_created\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-server-6ff784d758-mrg7n\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"a43b9675-fdba-4957-ab2b-41d084fe86c7\"}" timestamp=1689547061568
Jul 17 03:38:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:50.571Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_drop_total\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"veth83674fd8\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547061797
Jul 17 03:38:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:50.584Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db197\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689547066625
Jul 17 03:38:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:50.611Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_status_succeeded\", asset=\"twa-tsgx10\", asset_id=\"109\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", job_name=\"helm-install-cm-policy-local-cache\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689547059668
Jul 17 03:38:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:50.615Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"twa-tsgx10\", asset_id=\"109\", cpu=\"121\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547061797
Jul 17 03:38:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:50.652Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_resident_memory_bytes\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"mrzcpd.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689547066554
Jul 17 03:38:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:50.657Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"twa-tsgx10\", asset_id=\"109\", cpu=\"8\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547061797
Jul 17 03:38:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:50.674Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_scaling_frequency_max_hertz\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"81\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547066709
Jul 17 03:38:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:50.818Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_load_average_10s\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"prometheus-server-configmap-reload\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689547054562
Jul 17 03:38:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:50.839Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_ulimits_soft\", asset=\"twa-tsgx10\", asset_id=\"109\", container=\"lb-tcp-443\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx10\", endpoint_id=\"459\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", ulimit=\"max_open_files\"}" timestamp=1689547043811
Jul 17 03:38:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:50.869Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_restart_policy\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-server-6ff784d758-mrg7n\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", type=\"Always\", uid=\"a43b9675-fdba-4957-ab2b-41d084fe86c7\"}" timestamp=1689547061568
Jul 17 03:38:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:50.935Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_cpu_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-podbc493ff5_41bc_461a_a60b_e1fa21aeebe7.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"slice\"}" timestamp=1689547066554
Jul 17 03:38:50 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:50.961Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"28\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547066709
Jul 17 03:38:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:51.003Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_receive_errors_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", interface=\"bond0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689547065286
Jul 17 03:38:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:51.014Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"dev-disk-by\\\\x2did-scsi\\\\x2d362cea7f08346c1002c270339b2b39591\\\\x2dpart5.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"device\"}" timestamp=1689547066554
Jul 17 03:38:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:51.051Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_write_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"/run\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689547065286
Jul 17 03:38:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:51.072Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"go_mod_info\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"github.com/prometheus/common\", nz_agent_id=\"3\", program=\"systemd_exporter\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", version=\"v0.9.1\"}" timestamp=1689547066554
Jul 17 03:38:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:51.089Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"twa-tsgx08\", asset_id=\"105\", cpu=\"76\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547066709
Jul 17 03:38:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:51.090Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_changes_total\", asset=\"msh-dellr740-olap003\", asset_id=\"15\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"docker0\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-olap003\", endpoint_id=\"235\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"yarn_resourcemanager,zookeeper,nacos,kafka,spark,druid_data,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689547069713
Jul 17 03:38:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:51.164Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_compressed_total\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", device=\"virtio_dign_s\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547066709
Jul 17 03:38:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:51.180Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"sapp4_Dup_Key_Err\", app_name=\"sapp4\", asset=\"twa-tsgx08\", asset_id=\"105\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"traffic-engine\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-0\", vsysId=\"1\"}" timestamp=1689547058548
Jul 17 03:38:51 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:51.197Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_swap_limit_bytes\", asset=\"twa-tsgx08\", asset_id=\"105\", container=\"prometheus-alertmanager\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx08\", endpoint_id=\"401\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689547067561
Jul 17 03:38:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:52.466Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", cpu=\"84\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547069337
Jul 17 03:38:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:52.529Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_compressed_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens3f1np1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547069337
Jul 17 03:38:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:52.611Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", cpu=\"25\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547069337
Jul 17 03:38:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:52.624Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_compressed_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"ens4f1np1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547069337
Jul 17 03:38:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:52.825Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_scrape_collector_success\", asset=\"pcap-tsgx11\", asset_id=\"115\", collector=\"edac\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547069337
Jul 17 03:38:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:52.865Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_process_resident_memory_bytes\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"gpm.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\"}" timestamp=1689547065496
Jul 17 03:38:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:52.895Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_memory_usage_bytes\", asset=\"pcap-tsgx11\", asset_id=\"115\", container=\"coredns\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689547065341
Jul 17 03:38:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:52.911Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Running\", pod=\"helm-install-systemd-exporter-zlcr2\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"1df8c801-719d-43fa-af7f-961feb6e8372\"}" timestamp=1689547064478
Jul 17 03:38:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:52.925Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_cpu_user_seconds_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", container=\"prometheus-server\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689547062085
Jul 17 03:38:52 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:52.929Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"pcap-tsgx11\", asset_id=\"115\", condition=\"true\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"prometheus-server-6ff784d758-nghpf\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"21f91134-966a-424e-a89e-96c924c64f07\"}" timestamp=1689547064478
Jul 17 03:38:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:53.006Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_created\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"tsg-traffic-engine-numa-1-firewall\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"default\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689547064478
Jul 17 03:38:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:53.024Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas_ready\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", deployment=\"traefik\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689547064478
Jul 17 03:38:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:53.046Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.grpc.v1.cri-sandboxes-31e27c853a1da6df5a9c1323e8649e82872745922014f93e9d784a9eb440c1b0-shm.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689547072135
Jul 17 03:38:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:53.178Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_size_bytes\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda5\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", fstype=\"ext4\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/data\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547068328
Jul 17 03:38:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:53.224Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"msh-tsgx08\", asset_id=\"65\", condition=\"unknown\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"svclb-traefik-ab31cfb4-hhks6\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"44ba41ae-6294-4471-a8bc-5c98a2a4eb5d\"}" timestamp=1689547068950
Jul 17 03:38:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:53.241Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_max_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"50\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689547069337
Jul 17 03:38:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:53.266Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"data-var-lib-kubelet-pods-cd26913f\\\\x2dd130\\\\x2d4cb7\\\\x2da453\\\\x2d357d44e268db-volumes-kubernetes.io\\\\x7eprojected-kube\\\\x2dapi\\\\x2daccess\\\\x2d9zl28.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689547071179
Jul 17 03:38:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:53.401Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_edac_csrow_correctable_errors_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", controller=\"0\", csrow=\"unknown\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547069337
Jul 17 03:38:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:53.427Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_packets_total\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"cadvisor\", interface=\"eno1\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689547053538
Jul 17 03:38:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:53.561Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_processed_total\", asset=\"msh-tsgx08\", asset_id=\"65\", cpu=\"34\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547068328
Jul 17 03:38:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:53.576Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_total_error_replies\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689547070108
Jul 17 03:38:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:53.588Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_carrier_changes_total\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens8f0np0\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547068328
Jul 17 03:38:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:53.599Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-podae4b2545_b56f_4cc8_a1e1_3e91af1de5e6.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"slice\"}" timestamp=1689547065496
Jul 17 03:38:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:53.620Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"tmpfs\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/var/lib/kubelet/pods/4aff81f6-8efe-43da-b4c0-830d2bd4ff96/volumes/kubernetes.io~projected/kube-api-access-bpskd\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547069337
Jul 17 03:38:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:53.663Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Succeeded\", pod=\"helm-install-configmap-reloader-ftgrk\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"25126609-0484-4b4a-9130-b4002549c291\"}" timestamp=1689547062058
Jul 17 03:38:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:53.717Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_schedstat_waiting_seconds_total\", asset=\"msh-tsgx12\", asset_id=\"113\", cpu=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547063391
Jul 17 03:38:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:53.733Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx11\", asset_id=\"115\", cpu=\"92\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", mode=\"irq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547069337
Jul 17 03:38:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:53.783Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"pcap-tsgx11\", asset_id=\"115\", container=\"metrics-server\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"running\"}" timestamp=1689547069334
Jul 17 03:38:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:53.802Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"pcap-tsgx11\", asset_id=\"115\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx11\", endpoint_id=\"485\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Running\", pod=\"svclb-traefik-644fb0c0-7cq8s\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"ea6cc882-864f-480f-b3a6-4af3bfdaa023\"}" timestamp=1689547064478
Jul 17 03:38:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:53.802Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cpu=\"49\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547059409
Jul 17 03:38:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:53.810Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_sockets\", asset=\"pcap-tsgx07\", asset_id=\"85\", container=\"firewall\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689547065572
Jul 17 03:38:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:53.818Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_dropped_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cpu=\"58\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547059409
Jul 17 03:38:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:53.851Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", db=\"db153\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689547065184
Jul 17 03:38:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:53.871Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-ens7f1np1.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689547072135
Jul 17 03:38:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:53.918Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_scheduled\", asset=\"msh-tsgx08\", asset_id=\"65\", condition=\"true\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"local-path-provisioner-957fdf8bc-x869l\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"b9a87a56-93cb-41e5-883f-d940809bc904\"}" timestamp=1689547068950
Jul 17 03:38:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:53.959Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_files\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"shm\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", fstype=\"tmpfs\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/run/k3s/containerd/io.containerd.grpc.v1.cri/sandboxes/2d58616a1e2ba5df115c2438e56048be6927c43a5371eb4cc637d41e744d4d5b/shm\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547068328
Jul 17 03:38:53 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:53.971Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_multicast_total\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth8a8c0c8f\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547068328
Jul 17 03:38:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:54.018Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx08\", asset_id=\"65\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db228\", endpoint=\"TSG-OS-msh-tsgx08\", endpoint_id=\"433\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689547070108
Jul 17 03:38:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:54.037Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cpu=\"67\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547059409
Jul 17 03:38:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:54.071Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_swap_limit_bytes\", asset=\"pcap-tsgx07\", asset_id=\"85\", container=\"telegraf-security\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", serviceFunction=\"tsg-traffic-engine-numa-1\"}" timestamp=1689547070608
Jul 17 03:38:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:54.084Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_tasks_state\", asset=\"msh-tsgx12\", asset_id=\"113\", container=\"coredns\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", state=\"iowaiting\"}" timestamp=1689547045782
Jul 17 03:38:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:54.147Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_port_data_received_bytes_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"mlx5_3\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547059409
Jul 17 03:38:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:54.382Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db224\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689547058659
Jul 17 03:38:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:54.424Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys_expiring\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db241\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689547058659
Jul 17 03:38:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:54.513Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.runtime.v2.task-k8s.io-437d59346f57352ebc07722f8c2b0389dac5d783ae16e5514c1873ee111b6728-rootfs.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"mount\"}" timestamp=1689547071179
Jul 17 03:38:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:54.644Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx12\", asset_id=\"113\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx12\", endpoint_id=\"441\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-rdma-devices-mlx5_9:1-umad.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"device\"}" timestamp=1689547071179
Jul 17 03:38:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:54.691Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_blkio_device_usage_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", container=\"exporter-merger\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/dev/sda\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"cadvisor\", major=\"8\", minor=\"0\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operation=\"Total\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689547055246
Jul 17 03:38:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:54.833Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cpu=\"91\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547059409
Jul 17 03:38:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:54.860Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cpu=\"90\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", mode=\"user\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547059409
Jul 17 03:38:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:54.892Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_fifo_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth40581f6b\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547059409
Jul 17 03:38:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:54.932Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_errs_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"veth55feb6e8\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547059409
Jul 17 03:38:54 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:54.945Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_drop_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"eno35np2\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547059409
Jul 17 03:38:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:55.247Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_fs_writes_merged_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"/run\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689547065443
Jul 17 03:38:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:55.280Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_memory_AnonPages_bytes\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547059409
Jul 17 03:38:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:55.301Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_commands_failed_calls_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", cmd=\"expire\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689547065184
Jul 17 03:38:55 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:55.662Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_network_transmit_packets_dropped_total\", asset=\"pcap-tsgx07\", asset_id=\"85\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx07\", endpoint_id=\"479\", exported_job=\"cadvisor\", interface=\"eno35np2\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689547065443
Jul 17 03:38:56 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:56.282Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_ready\", asset=\"pcap-tsgx03\", asset_id=\"73\", condition=\"false\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"systemd-exporter-7f549d9485-v5fwm\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"de5d06af-7c33-48a2-ba8b-4ec77a0b93a3\"}" timestamp=1689547073389
Jul 17 03:38:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:57.639Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_job_labels\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", job_name=\"helm-install-configmap-reloader\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689547073389
Jul 17 03:38:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:57.709Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_infiniband_unicast_packets_transmitted_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", datacenter=\"PCAP\", datacenter_id=\"1\", device=\"mlx5_6\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", port=\"1\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547065883
Jul 17 03:38:57 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:57.742Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"pcap-tsgx03\", asset_id=\"73\", cpu=\"5\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"TSG-OS-pcap-tsgx03\", endpoint_id=\"467\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547065883
Jul 17 03:38:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:59.702Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_ips\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", ip=\"10.42.1.134\", ip_family=\"4\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", pod=\"cm-policy-local-cache-redis-master-69967b5879-kmxnw\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"afda43ab-d2e7-4daa-bc44-9edf3c57be96\"}" timestamp=1689547067425
Jul 17 03:38:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:59.734Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db89\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689547065071
Jul 17 03:38:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:59.750Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"124\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547069083
Jul 17 03:38:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:59.752Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-subsystem-net-devices-ens6f1np1.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"device\"}" timestamp=1689547064879
Jul 17 03:38:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:59.768Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_iface_id\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"ens4f2np2\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547069083
Jul 17 03:38:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:59.794Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", db=\"db103\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689547065071
Jul 17 03:38:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:59.857Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-update-done.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"service\"}" timestamp=1689547064879
Jul 17 03:38:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:59.872Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_sockets\", asset=\"msh-tsgx01\", asset_id=\"51\", container=\"local-path-provisioner\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689547061159
Jul 17 03:38:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:59.915Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"phy_rx_nobuf_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", dev=\"ens7f0np0\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"mrzcpd-exporter\"}" timestamp=1689547070377
Jul 17 03:38:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:59.941Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_allocator_active_bytes\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689547065071
Jul 17 03:38:59 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:38:59.998Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_filesystem_device_error\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"/dev/sda5\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", fstype=\"ext4\", module=\"TSG-OS\", module_id=\"77\", mountpoint=\"/data/var/lib/kubelet/pods/55b23504-098e-4605-9a10-91ab01c5f1f9/volume-subpaths/config-volume/telegraf-security/1\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547069083
Jul 17 03:39:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:39:00.137Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"14\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547069083
Jul 17 03:39:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:39:00.436Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_receive_packets_total\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth9d7eadc6\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547069083
Jul 17 03:39:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:39:00.450Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_info\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", mount_type=\"tmpfs\", name=\"tmp.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"mount\"}" timestamp=1689547064879
Jul 17 03:39:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:39:00.467Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"crond.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"service\"}" timestamp=1689547064879
Jul 17 03:39:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:39:00.542Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_guest_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"94\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547069083
Jul 17 03:39:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:39:00.601Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"119\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", mode=\"softirq\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547069083
Jul 17 03:39:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:39:00.637Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_softnet_times_squeezed_total\", asset=\"msh-tsgx01\", asset_id=\"51\", cpu=\"66\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547069083
Jul 17 03:39:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:39:00.936Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_deployment_status_replicas_ready\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", deployment=\"ipmi-exporter\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689547067425
Jul 17 03:39:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:39:00.970Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Pending\", pod=\"helm-install-cm-policy-local-cache-9rh48\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"592a0d86-ef77-4a04-b590-aa94c9fe12b4\"}" timestamp=1689547067425
Jul 17 03:39:00 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:39:00.987Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_spec_replicas\", asset=\"msh-tsgx01\", asset_id=\"51\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"metrics-server-54dc485875\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689547067425
Jul 17 03:39:01 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:39:01.318Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"container_spec_memory_reservation_limit_bytes\", asset=\"msh-tsgx01\", asset_id=\"51\", container=\"prometheus-alertmanager\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx01\", endpoint_id=\"399\", exported_job=\"cadvisor\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\"}" timestamp=1689547074593
Jul 17 03:39:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:39:02.831Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", phase=\"Unknown\", pod=\"traefik-64f55bb67d-lpmmw\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"11efef4a-05e0-44ee-8681-0e306eccb51c\"}" timestamp=1689547071412
Jul 17 03:39:02 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:39:02.852Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_reason\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"kube-system\", nz_agent_id=\"3\", pod=\"svclb-traefik-1a2acd8b-r8v4z\", project=\"TSG-OS\", project_id=\"7\", reason=\"Evicted\", service=\"prometheus-kube-state-metrics\", uid=\"80a49efb-cd3a-4d5f-8345-2cc7583d6815\"}" timestamp=1689547071412
Jul 17 03:39:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:39:03.019Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_info\", address=\"de:cd:3c:99:7b:a9\", asset=\"msh-tsgx02\", asset_id=\"63\", broadcast=\"ff:ff:ff:ff:ff:ff\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"veth74f1e741\", duplex=\"full\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", operstate=\"up\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547078318
Jul 17 03:39:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:39:03.034Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_replicaset_owner\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", owner_is_controller=\"true\", owner_kind=\"Deployment\", owner_name=\"prometheus-alertmanager\", project=\"TSG-OS\", project_id=\"7\", replicaset=\"prometheus-alertmanager-6df7f457b6\", service=\"prometheus-kube-state-metrics\"}" timestamp=1689547071412
Jul 17 03:39:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:39:03.053Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx02\", asset_id=\"63\", cpu=\"81\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547078318
Jul 17 03:39:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:39:03.131Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-pod87913277_0e7d_4545_8517_0068668d7130.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"slice\"}" timestamp=1689547068447
Jul 17 03:39:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:39:03.152Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cooling_device_cur_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"104\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\", type=\"Processor\"}" timestamp=1689547078318
Jul 17 03:39:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:39:03.228Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_network_transmit_bytes_total\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", device=\"vethad44a482\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547078318
Jul 17 03:39:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:39:03.247Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"kubepods-besteffort-pod6c93731d_28e4_47e7_a070_2872ea9641a7.slice\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"deactivating\", type=\"slice\"}" timestamp=1689547068447
Jul 17 03:39:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:39:03.459Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"msh-tsgx02\", asset_id=\"63\", cpu=\"34\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", mode=\"steal\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547078318
Jul 17 03:39:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:39:03.803Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"kube_pod_status_phase\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", namespace=\"tsg-os-system\", nz_agent_id=\"3\", phase=\"Succeeded\", pod=\"mrzcpd-exporter-5cf7f967cc-9zrnz\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-kube-state-metrics\", uid=\"0b1be0e3-e71c-4f3b-83a7-1d29910105b8\"}" timestamp=1689547071412
Jul 17 03:39:03 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:39:03.927Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"sys-devices-virtual-net-veth2d52e57c.device\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"device\"}" timestamp=1689547068447
Jul 17 03:39:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:39:04.008Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"prometheus_target_scrape_pool_sync_total\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", instance=\"10.10.20.159:10090\", scrape_job=\"259\", type=\"global\"}" timestamp=1689547082989
Jul 17 03:39:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:39:04.025Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"pcap-dellr740-dt002\", asset_id=\"33\", cpu=\"6\", datacenter=\"PCAP\", datacenter_id=\"1\", endpoint=\"OLAP-Node-Exporter-pcap-dellr740-dt002\", endpoint_id=\"253\", mode=\"user\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689547082497
Jul 17 03:39:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:39:04.033Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"net_conntrack_dialer_conn_failed_total\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", dialer_name=\"103\", instance=\"10.10.20.159:10090\", reason=\"refused\", type=\"global\"}" timestamp=1689547082989
Jul 17 03:39:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:39:04.104Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"net_conntrack_dialer_conn_failed_total\", agent_name=\"msh-kvm-nz001\", application=\"prometheus\", datacenter=\"MSH\", dialer_name=\"41\", instance=\"10.10.20.159:10090\", reason=\"timeout\", type=\"global\"}" timestamp=1689547082989
Jul 17 03:39:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:39:04.128Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"msh-tsgx02\", asset_id=\"63\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"TSG-OS-msh-tsgx02\", endpoint_id=\"431\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"run-k3s-containerd-io.containerd.grpc.v1.cri-sandboxes-202be1d5820ca6fcad88415c7e5ac2e0f59b2214769b6b6969372b483df27e0e-shm.mount\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"failed\", type=\"mount\"}" timestamp=1689547068447
Jul 17 03:39:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:39:04.244Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_scaling_frequency_min_hertz\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", cpu=\"22\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-dt001\", endpoint_id=\"225\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689547082988
Jul 17 03:39:04 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:39:04.441Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_memory_VmallocUsed_bytes\", asset=\"msh-dellr740-dt001\", asset_id=\"5\", datacenter=\"MSH\", datacenter_id=\"3\", endpoint=\"OLAP-Node-Exporter-msh-dellr740-dt001\", endpoint_id=\"225\", module=\"OLAP-Node-Exporter\", module_id=\"51\", nz_agent_id=\"3\", olap_node_exporter=\"clickhouse_data,kafka,yarn_resourcemanager,gohangout,zookeeper,hbase_master,hadoop_namenode,hos_load_balancer,hos,hadoop_datanode,hbase_region,yarn_nodemanager\", project=\"OLAP\", project_id=\"3\"}" timestamp=1689547082988
Jul 17 03:39:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:39:06.789Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_connections_received_total\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689547078290
Jul 17 03:39:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:39:06.828Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cpu=\"120\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", mode=\"system\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547083910
Jul 17 03:39:06 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:39:06.835Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_tasks_max\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-binfmt.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", type=\"service\"}" timestamp=1689547073582
Jul 17 03:39:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:39:07.039Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_commands_rejected_calls_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cmd=\"zscore\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689547078290
Jul 17 03:39:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:39:07.054Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"redis_db_keys\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", db=\"db200\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"cm-policy-local-cache-redis-metrics\"}" timestamp=1689547078290
Jul 17 03:39:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:39:07.060Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"systemd-machine-id-commit.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"active\", type=\"service\"}" timestamp=1689547073582
Jul 17 03:39:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:39:07.107Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"systemd_unit_state\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", name=\"unbound-anchor.service\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"systemd-exporter\", state=\"inactive\", type=\"service\"}" timestamp=1689547073582
Jul 17 03:39:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:39:07.166Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cpu=\"69\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", mode=\"idle\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547083910
Jul 17 03:39:07 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:39:07.227Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_memory_Hugepagesize_bytes\", asset=\"twa-tsgx01\", asset_id=\"91\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547083910
Jul 17 03:39:08 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:39:08.012Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"node_cpu_seconds_total\", asset=\"twa-tsgx01\", asset_id=\"91\", cpu=\"86\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"TSG-OS-twa-tsgx01\", endpoint_id=\"443\", exported_job=\"exporter\", mode=\"nice\", module=\"TSG-OS\", module_id=\"77\", nz_agent_id=\"3\", project=\"TSG-OS\", project_id=\"7\", service=\"prometheus-node-exporter\"}" timestamp=1689547083910
Jul 17 03:39:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:39:10.545Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_storeFileSize\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"time\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"session_record_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"353664fe0caa12c046540001fa2673bf\", table=\"index_time_session_record_hos_bucket\", type=\"index\"}" timestamp=1689547088804
Jul 17 03:39:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:39:10.580Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_avgStoreFileAge\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", module=\"DC-HBase\", module_id=\"3\", namespace=\"SYSTEM\", nz_agent_id=\"3\", project=\"OLAP\", project_id=\"3\", region=\"40e370b111bd00ac33f374bfbafde582\", table=\"LOG\", type=\"data\"}" timestamp=1689547088804
Jul 17 03:39:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:39:10.727Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_getCount\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"filename\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"firewall_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"700a07cb70ff9f2c61808c3e4df6599d\", table=\"index_filename_firewall_hos_bucket\", type=\"index\"}" timestamp=1689547088804
Jul 17 03:39:10 msh-kvm-nz001 prometheus[1044]: ts=2023-07-16T22:39:10.759Z caller=write_handler.go:109 level=error component=web msg="Out of order sample from remote write" err="out of order sample" series="{__name__=\"Hadoop_HBase_metric_numBytesCompactedCount\", asset=\"twa-dellr740-dt001\", asset_id=\"37\", datacenter=\"TWA\", datacenter_id=\"5\", endpoint=\"DC-HBase-twa-dellr740-dt001-HregionServer\", endpoint_id=\"159\", indextype=\"partfile\", module=\"DC-HBase\", module_id=\"3\", namespace=\"default\", nz_agent_id=\"3\", parent=\"assessment_hos_bucket\", project=\"OLAP\", project_id=\"3\", region=\"c16a0ad8b16e365f0379acba25fc24b7\", table=\"index_partfile_assessment_hos_bucket\", type=\"index\"}" timestamp=1689547088804